Retry NAV API operations on timeout

bin/kindnavsync

@@ -18,12 +18,14 @@ import argparse
 
 from slack_log_handler import SlackLogHandler
 
+from kindnavsync.errors import retry_on_timeout
 from kindnavsync.navapi import NAVAPI, ObjectNotFound, ClientError
 
 KIND_SERVICE_URL = "https://kind.uninett.no/autoregister/utstyr_eksport.json"
 KIND_UTSTYR_URL = "https://kind.uninett.no/utstyr/edit_utstyr.html?id={id}"
 NAV_BOX_URL = "https://uninav.uninett.no/ipdevinfo/{sysname}/"
 NAV_API_URL = "http://localhost/api/1"
+NAV_API_TIMEOUT = 10.0
 
 DEFAULT_SNMP_COMMUNITY = "public"
 DEFAULT_SNMP_VERSION = "2"
@@ -412,8 +414,9 @@ class NAVBox(dict):
 
 def update_nav(kind_devices, api_url, api_token, dry_run=False):
     kind_by_ip = {ip: dev for dev in kind_devices.values() for ip in dev.ipaddrs}
-    nav_api = NAVAPI(url=api_url, auth_token=api_token)
-    netboxes = set(NAVBox(n) for n in nav_api.list_netboxes())
+    nav_api = NAVAPI(url=api_url, auth_token=api_token, timeout=NAV_API_TIMEOUT)
+    list_netboxes = retry_on_timeout(retries=3, delay=1)(nav_api.list_netboxes)
+    netboxes = set(NAVBox(n) for n in list_netboxes())
 
     matched_netboxes = set()
     for netbox in netboxes:
@@ -466,6 +469,7 @@ def insert_into_nav(nav_api, kind_devices):
             LOG.exception("Kunne ikke lagre %s: %s", dev, err)
 
 
+@retry_on_timeout(retries=3, delay=1)
 def insert_one_into_nav(nav_api, kind_device):
     """
     Inserts one Kind device into NAV.
@@ -517,6 +521,7 @@ def insert_one_into_nav(nav_api, kind_device):
 
 
 @functools.lru_cache(1)
+@retry_on_timeout(retries=3, delay=1)
 def find_or_create_default_management_profile(nav_api):
     """
     :type nav_api: kindnavsync.navapi.NAVAPI
@@ -545,6 +550,7 @@ def update_existing_in_nav(nav_api, kind_devices):
         update_device_in_nav_if_changed(nav_api, device)
 
 
+@retry_on_timeout(retries=3, delay=1)
 def update_device_in_nav_if_changed(nav_api, kind_device):
     """Updates kind_device in NAV if its details have changed.
 
@@ -595,7 +601,8 @@ def remove_from_nav(nav_api, deleteable_boxes):
     deleteable_ids = tuple(n.id for n in deleteable_boxes)
     LOG.info("Fjerner %d bokser fra NAV", len(deleteable_boxes))
     for netbox_id in deleteable_ids:
-        nav_api.delete_netbox(netbox_id)
+        delete = retry_on_timeout(retries=1, delay=1)(nav_api.delete_netbox)
+        delete(netbox_id)
 
 
 def can_be_deleted(netbox):