Commit 27fdb8db authored by Sigmund Augdal's avatar Sigmund Augdal

Speed up duplicate line detection significantly by using an OrderedDict to keep output lines

parent fca5ebd2
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function, unicode_literals
from collections import OrderedDict
import subprocess
import logging
from nova_router import security_groups, setupLogfile, send_stat, etcd_connect
......@@ -73,13 +74,13 @@ class Generator(object):
self.ip_cidr_groups = {}
self.generation = 0
self.serial = 0
self.output_data = []
self.output_data = OrderedDict()
self.open_groups = set()
self.group_members = {}
def output(self, line):
if line not in self.output_data:
self.output_data.append(line)
self.output_data[line] = 1
else:
logging.warning("duplicate ipset entry: %s", line)
......@@ -229,7 +230,7 @@ class Generator(object):
self.ip_cidr_groups = {}
self.serial = 0
self.open_groups = set()
self.output_data = []
self.output_data = OrderedDict()
self.group_members = {}
if not self.noop:
......@@ -256,7 +257,7 @@ class Generator(object):
for group_id, group in sorted(groups.items(), key=lambda x: x[0]):
self.process_security_group(group_id, group['name'])
output_data = [line for line in self.output_data if self.group_members[line.split(" ")[1]]]
output_data = [line for line in self.output_data.keys() if self.group_members[line.split(" ")[1]]]
if not self.noop:
with tempfile.TemporaryFile() as output_file:
output_file.write("\n".join(output_data))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment