Handle timeouts properly
The API calls used by kindnavsync may intermittently time out. This causes massive heaps of traceback messages to be output - and if running in cron, this output is sent as annoying e-mails.
These errors should be handled more gracefully. A combined solution would:
- Retry a few times before giving up entirely
- Catch timeout errors and output a much more succinct error message to the user (e.g. "The NAV API timed out")
An example:
2020-09-08 16:00:25,145 [ERROR] [simple_rest_client.decorators] HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1236, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.5/socket.py", line 576, in readinto
return self._sock.recv_into(b)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 326, in recv_into
raise timeout("The read operation timed out")
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/util/retry.py", line 400, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 423, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 331, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/decorators.py", line 30, in wrapper
response = f(*args, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/request.py", line 23, in make_request
**request.kwargs
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1236, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.5/socket.py", line 576, in readinto
return self._sock.recv_into(b)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 326, in recv_into
raise timeout("The read operation timed out")
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/util/retry.py", line 400, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 423, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 331, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/decorators.py", line 30, in wrapper
response = f(*args, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/request.py", line 23, in make_request
**request.kwargs
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/mvold/.local/bin/kindnavsync", line 654, in <module>
main()
File "/home/mvold/.local/bin/kindnavsync", line 77, in main
dry_run=args.dry_run,
File "/home/mvold/.local/bin/kindnavsync", line 416, in update_nav
netboxes = set(NAVBox(n) for n in nav_api.list_netboxes())
File "/home/mvold/.local/lib/python3.5/site-packages/kindnavsync/navapi.py", line 41, in list_netboxes
response = self.api.netbox.list(params={"page_size": 1000})
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/resource.py", line 110, in action_method
return make_request(self.session, request)
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/decorators.py", line 33, in wrapper
raise ClientConnectionError() from exc
simple_rest_client.exceptions.ClientConnectionError