Add find-dead-trapper-items subcommand.

zabbix-cli.py

@@ -5,6 +5,7 @@ import getpass
 import json
 import os
 import requests
+import statistics
 import sys
 import time
 
@@ -265,6 +266,46 @@ def get_triggers(args):
             print(" - {severity} NOHOST: {trigger}".format(trigger=description, severity=severity))
 
 
+def find_dead_trapper_items(args):
+    for host in invoke('host.get'):
+        for item in invoke('item.get', hostids=host['hostid']):
+            if int(item['type']) != ITEM_TYPE_MAP['trapper']:
+                continue
+            if item['status'] == '1': # disabled
+                continue
+            # Ignore some items that are known to only report in some special cases
+            if item['key_'] == 'cron.job.failure' or item['key_'].endswith('.last_error'):
+                continue
+
+            last_timestamp = None
+            prev_timestamp = None
+            intervals = []
+            for value in invoke('history.get', history=int(item['value_type']), itemids=int(item['itemid']), sortfield='clock', sortorder='DESC', limit=50):
+                value_timestamp = int(value['clock']) + int(value['ns']) / 1e12
+                if prev_timestamp:
+                    intervals.append(prev_timestamp - value_timestamp)
+                if not last_timestamp or last_timestamp < value_timestamp:
+                    last_timestamp = value_timestamp
+                prev_timestamp = value_timestamp
+
+            if len(intervals) < 2:
+                continue
+            mean_interval = statistics.mean(intervals)
+            stdev_interval = statistics.stdev(intervals)
+            delay_threshold = mean_interval + 4 * stdev_interval
+            current_delay = time.time() - last_timestamp
+            if current_delay > delay_threshold:
+                print('https://zabbix.uninett.no/zabbix/items.php?form=update&hostid={hostid}&itemid={itemid} {key} current_delay={current_delay} delay_threshold={delay_threshold} (mean={mean_interval} stdev={stdev_interval})'.format(
+                    hostid=item['hostid'],
+                    itemid=item['itemid'],
+                    key=item['key_'],
+                    current_delay=current_delay,
+                    delay_threshold=delay_threshold,
+                    mean_interval=mean_interval,
+                    stdev_interval=stdev_interval,
+                ))
+
+
 def parse_args():
     parser = argparse.ArgumentParser('Zabbix cmd line client')
     subparsers = parser.add_subparsers(help='Command', dest='command')
@@ -305,6 +346,8 @@ def parse_args():
     triggers = subparsers.add_parser('get-triggers', help="List triggers")
     triggers.add_argument('--severity', choices=SEVERITIES.keys(), help="Minimum severity to include")
 
+    find_dead_trapper_items = subparsers.add_parser('find-dead-trapper-items', help="Find trapper items no longer receiving data")
+
     return parser.parse_args()
 
 
@@ -332,6 +375,8 @@ def main():
         return find_empty(args)
     elif args.command == 'get-triggers':
         return get_triggers(args)
+    elif args.command == 'find-dead-trapper-items':
+        return find_dead_trapper_items(args)
     else:
         raise NotImplementedError('Unknown command: {command}'.format(command=args.command))