You need to sign in or sign up before continuing.
Handle timeouts properly
The API calls used by kindnavsync may intermittently time out. This causes massive heaps of traceback messages to be output - and if running in cron, this output is sent as annoying e-mails.
These errors should be handled more gracefully. A combined solution would:
- Retry a few times before giving up entirely
- Catch timeout errors and output a much more succinct error message to the user (e.g. "The NAV API timed out")
An example:
2020-09-08 16:00:25,145 [ERROR] [simple_rest_client.decorators] HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1236, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.5/socket.py", line 576, in readinto
return self._sock.recv_into(b)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 326, in recv_into
raise timeout("The read operation timed out")
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/util/retry.py", line 400, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 423, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 331, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/decorators.py", line 30, in wrapper
response = f(*args, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/request.py", line 23, in make_request
**request.kwargs
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1236, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.5/socket.py", line 576, in readinto
return self._sock.recv_into(b)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/contrib/pyopenssl.py", line 326, in recv_into
raise timeout("The read operation timed out")
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/util/retry.py", line 400, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 423, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/home/mvold/.local/lib/python3.5/site-packages/urllib3/connectionpool.py", line 331, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/decorators.py", line 30, in wrapper
response = f(*args, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/request.py", line 23, in make_request
**request.kwargs
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/home/mvold/.local/lib/python3.5/site-packages/requests/adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='uninav.uninett.no', port=443): Read timed out. (read timeout=3)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/mvold/.local/bin/kindnavsync", line 654, in <module>
main()
File "/home/mvold/.local/bin/kindnavsync", line 77, in main
dry_run=args.dry_run,
File "/home/mvold/.local/bin/kindnavsync", line 416, in update_nav
netboxes = set(NAVBox(n) for n in nav_api.list_netboxes())
File "/home/mvold/.local/lib/python3.5/site-packages/kindnavsync/navapi.py", line 41, in list_netboxes
response = self.api.netbox.list(params={"page_size": 1000})
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/resource.py", line 110, in action_method
return make_request(self.session, request)
File "/home/mvold/.local/lib/python3.5/site-packages/simple_rest_client/decorators.py", line 33, in wrapper
raise ClientConnectionError() from exc
simple_rest_client.exceptions.ClientConnectionError