Commit 9e6b4f15 authored by Morten Knutsen's avatar Morten Knutsen

Rewrite to use python-etcd library instead of etcd-py.

parent e55ab228
......@@ -39,8 +39,9 @@ def main():
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(name)s %(levelname)s %(message)s')
etcd_client = etcd.Etcd("nova-gw1.bs.unsi.no", ssl_key=conf.get('etcd', 'key'),
ssl_cert=conf.get('etcd', 'cert'), verify=conf.get('etcd', 'cacert'))
etcd_client = etcd.Client(host="nova-gw1.bs.unsi.no", cert=(conf.get('etcd', 'cert'),
conf.get('etcd', 'key')),
ca_cert=conf.get('etcd', 'cacert'), protocol='https')
vmm_client = odata.VMMClient(conf.get('spf', 'vmm_baseurl'), conf.get('spf', 'username'),
conf.get('spf', 'password'))
pdns_url = conf.get('pdns', 'url')
......@@ -51,7 +52,7 @@ def main():
vmnet = nic.properties["VMNetworkId"]
if mac is not None and (vmnet == uuid.UUID("5cc21a4c-a5b9-413d-b607-3ce7020c8b98") or vmnet is None):
nics.add(mac.lower())
for entry in etcd_client.list("/nova/iaas/instances"):
for entry in etcd_client.read("/nova/iaas/instances").children:
if not entry.dir:
next
mac = entry.key.split("/")[-1].lower()
......
......@@ -20,7 +20,7 @@ HOSTS_FILE = "/etc/dhcp/generated_hosts.conf"
def etcd_get_dict(etcd_client, prefix):
result = dict()
for key in etcd_client.list(prefix):
for key in etcd_client.read(prefix).children:
if not key.dir:
result[key.key.split('/')[-1]] = key.value
return result
......@@ -28,7 +28,7 @@ def etcd_get_dict(etcd_client, prefix):
def etcd_get_dirs(etcd_client, prefix):
result = set()
for key in etcd_client.list(prefix):
for key in etcd_client.read(prefix).children:
if key.dir:
result.add(key.key.split('/')[-1])
return result
......@@ -47,7 +47,9 @@ def format_ip(num):
class Generator(object):
def __init__(self, etcd_host, cert, key, cacert, logfile=None):
self.etcd_client = etcd.Etcd(etcd_host, ssl_key=key, ssl_cert=cert, verify=cacert)
self.etcd_client = etcd.Client(host="nova-gw1.bs.unsi.no", cert=(conf.get('etcd', 'cert'),
conf.get('etcd', 'key')),
ca_cert=conf.get('etcd', 'cacert'), protocol='https')
if logfile:
handler = logging.handlers.RotatingFileHandler(logfile,
maxBytes=10*1024**3, backupCount=5)
......@@ -101,18 +103,17 @@ class Generator(object):
def get_hosts(self):
index = None
hosts = {}
for entry in self.etcd_client.list("/nova/iaas/instances"):
for entry in self.etcd_client.read("/nova/iaas/instances").children:
if not entry.dir:
continue
if not index:
index = entry.index
index = entry.modifiedIndex
mac = entry.key.split("/")[-1]
try:
ipaddress = self.etcd_client.get(entry.key + "/ipv4").value
hosts[ipaddress] = mac
except etcd.EtcdError as ex:
if ex.args[0] != 100:
raise ex
except KeyError:
pass
return hosts, index
def generate_all(self):
......@@ -139,7 +140,7 @@ class Generator(object):
index = self.generate_all()
while True:
data = self.etcd_client.watch("/nova/iaas/instances", index+1)
logging.debug("new config index %d", data.index)
logging.debug("new config index %d", data.modifiedIndex)
time.sleep(1)
index = self.generate_all()
......
......@@ -23,7 +23,8 @@ IPTABLES_OPEN_GROUP = "-A FORWARD -m set --match-set {0} src -m set --match-set
class Generator(object):
def __init__(self, cert, key, cacert, logfile=None):
self.etcd_client = etcd.Etcd(ssl_key=key, ssl_cert=cert, verify=cacert)
self.etcd_client = etcd.Client(host="localhost", cert=(cert,key),
ca_cert=cacert, protocol='https')
if logfile:
handler = logging.handlers.RotatingFileHandler(logfile, maxBytes=10*1024**3,
backupCount=5)
......@@ -114,16 +115,15 @@ class Generator(object):
def get_addresses(self, addrtype):
addresses = {}
for entry in self.etcd_client.list("/nova/iaas/instances"):
for entry in self.etcd_client.read("/nova/iaas/instances").children:
if not entry.dir:
continue
mac = entry.key.split("/")[-1]
try:
ipaddr = self.etcd_client.get(entry.key + "/" + addrtype).value
addresses[mac] = ipaddr
except etcd.EtcdError as ex:
if ex.args[0] != 100:
raise ex
except KeyError:
pass
return addresses
def import_iptables(self, filename, output):
......@@ -189,18 +189,18 @@ class Generator(object):
return index
def main(self):
index = next(self.etcd_client.list("/nova/iaas")).index
index = next(self.etcd_client.read("/nova/iaas")).modifiedIndex
self.generate_all()
while True:
data = self.etcd_client.watch("/nova/iaas", index+1)
index = data.index
index = data.modifiedIndex
key = data.key # pylint: disable=E1101
if not key.startswith("nova/iaas/instances") and not \
key.startswith("nova/iaas/security_group"):
continue
logging.debug("new config index %d", data.index)
time.sleep(1)
index = next(self.etcd_client.list("/nova/iaas")).index
index = next(self.etcd_client.read("/nova/iaas")).modifiedIndex
self.generate_all()
......
......@@ -40,8 +40,8 @@ def unicast_mac(mac):
class NeighborCache(object):
def __init__(self, ssl_cert, ssl_key, cacert, interface, routers):
self.etcd_client = etcd.Etcd(ssl_key=ssl_key, ssl_cert=ssl_cert,
verify=cacert)
self.etcd_client = etcd.Client(host="localhost", cert=(ssl_cert,ssl_key),
ca_cert=cacert, protocol='https')
self.caches = ({}, {}, {})
self.ifindex = ifindex(interface)
self.ir = IPRoute()
......
from __future__ import absolute_import, division, print_function, unicode_literals
import etcd
import logging
......@@ -17,43 +16,36 @@ def get_ipaddress_from_mac(etcd_client, mac, addrtype):
mac = mac.lower()
try:
return etcd_client.get(etcd_key(mac, addrtype)).value
except etcd.EtcdError as ex:
if ex.args[0] == 100:
return None
raise ex
except KeyError:
pass
def get_mac_from_ipaddress(etcd_client, ipaddress, addrtype):
try:
for entry in etcd_client.list("nova/iaas/instances"):
for entry in etcd_client.read("nova/iaas/instances").children:
if entry.dir:
mac = entry.key.split("/")[-1]
try:
if etcd_client.get(entry.key + "/" + addrtype).value == ipaddress:
return mac
except etcd.EtcdError as ex:
if ex.args[0] != 100:
raise ex
except etcd.EtcdError as ex:
if ex.args[0] == 100:
return None
except KeyError:
pass
except KeyError:
return None
return None
def add_pair(etcd_client, mac, ipaddress, addrtype):
mac = mac.lower()
try:
etcd_client.testandset(etcd_key(mac, addrtype), "", ipaddress)
except etcd.EtcdError as ex:
if ex.args[0] == 101:
remote_addr = etcd_client.get(etcd_key(mac, addrtype)).value
if remote_addr != ipaddress:
log.warning("mac %s changed address to %s while we tried to set it to %s",
mac, remote_addr, ipaddress)
else:
pass # Other router did the update quicker than us
etcd_client.write(etcd_key(mac, addrtype), ipaddress, prevExist=False)
except KeyError: # Key exists
remote_addr = etcd_client.get(etcd_key(mac, addrtype)).value
if remote_addr != ipaddress:
log.warning("mac %s changed address to %s while we tried to set it to %s",
mac, remote_addr, ipaddress)
else:
raise ex
pass # Other router did the update quicker than us
def remove_pair(etcd_client, mac, ipaddress, addrtype):
......
from __future__ import absolute_import, division, print_function, unicode_literals
import etcd
import socket
import logging
import uuid
......@@ -46,7 +45,7 @@ def get_security_groups(etcd_client, cloudid=None):
cloudid = str(cloudid)
groups = {}
try:
for entry in etcd_client.list(security_group_key()):
for entry in etcd_client.read(security_group_key()).children:
if entry.dir:
group_id = entry.key.split("/")[-1]
try:
......@@ -56,9 +55,9 @@ def get_security_groups(etcd_client, cloudid=None):
continue
name = etcd_client.get(entry.key + "/name")
groups[group_id] = dict(name=name.value, cloudid=cloud)
except etcd.EtcdError:
except KeyError:
pass
except etcd.EtcdError:
except KeyError:
pass
return groups
......@@ -67,21 +66,17 @@ def get_group_rules(etcd_client, group_id):
group_key = security_group_key(group_id)
rules = []
try:
for entry in etcd_client.list(group_key + "/rules"):
for entry in etcd_client.read(group_key + "/rules").children:
if entry.dir:
rule = dict(id=entry.key.split("/")[-1])
for key in RULE_KEYS:
try:
rule[key] = etcd_client.get(entry.key + "/" + key).value
except etcd.EtcdError as ex:
if ex.args[0] != 100:
logging.exception("a")
except KeyError:
logging.exception("a")
if "protocol" in rule and "source_type" in rule and "destination_port" in rule:
rules.append(rule)
except etcd.EtcdError as ex:
if ex.args[0] != 100:
logging.exception("Failed getting group rules")
raise ex
except KeyError:
return []
rules.sort(key=lambda x: x['id'])
return rules
......@@ -91,9 +86,7 @@ def get_group_mode(etcd_client, group_id):
group_key = security_group_key(group_id)
try:
return etcd_client.get(group_key + "/mode").value
except etcd.EtcdError as ex:
if ex.args[0] != 100:
raise ex
except KeyError:
return "open"
......@@ -147,7 +140,7 @@ def add_rule(etcd_client, group_id, params):
source_security_group = params["source_security_group"]
try:
etcd_client.get(security_group_key(source_security_group) + "/name")
except etcd.EtcdError:
except KeyError:
raise ValueError("Invalid source security group: " + source_security_group)
etcd_client.set(rule_key + "/protocol", protocol)
......@@ -164,7 +157,8 @@ def add_rule(etcd_client, group_id, params):
def del_rule(etcd_client, group_id, rule_id):
rule_key = security_group_rule_key(group_id, rule_id)
for entry in etcd_client.list(rule_key):
# TODO: Use recursive delete
for entry in etcd_client.read(rule_key).children:
etcd_client.delete(entry.key)
......@@ -172,9 +166,7 @@ def get_group_members(etcd_client, group_id):
members_key = security_group_key(group_id) + "/members/by_mac"
try:
raw_members = etcd_client.get(members_key).value
except etcd.EtcdError as ex:
if ex.args[0] != 100:
raise ex
except KeyError:
return "", []
members = raw_members.split("\n")
members = [m for m in members if m]
......@@ -199,7 +191,7 @@ def add_mac_to_group(etcd_client, group_id, mac):
return
members.append(mac)
new_members = "\n".join(members)
etcd_client.testandset(members_key, raw_members, new_members)
etcd_client.test_and_set(members_key, new_members, raw_members)
def delete_mac_from_group(etcd_client, group_id, mac):
......@@ -213,4 +205,4 @@ def delete_mac_from_group(etcd_client, group_id, mac):
logging.debug("new members: %s", new_members)
if not new_members:
new_members = "\n"
etcd_client.testandset(members_key, raw_members, new_members)
etcd_client.test_and_set(members_key, new_members, raw_members)
......@@ -53,9 +53,9 @@ class TestSecurityGroups(object):
get_group_members.return_value = ("aa:bb:cc:dd:ee:ff", ["aa:bb:cc:dd:ee:ff"])
etcd_client = mock.MagicMock()
security_groups.add_mac_to_group(etcd_client, sec_group, "11:22:33:44:55:66")
etcd_client.testandset.assert_called_with(self.members_key,
"aa:bb:cc:dd:ee:ff",
"aa:bb:cc:dd:ee:ff\n11:22:33:44:55:66")
etcd_client.test_and_set.assert_called_with(self.members_key,
"aa:bb:cc:dd:ee:ff\n11:22:33:44:55:66",
"aa:bb:cc:dd:ee:ff")
etcd_client = mock.MagicMock()
security_groups.add_mac_to_group(etcd_client, sec_group, "aa:bb:cc:dd:ee:ff")
assert not etcd_client.get.called
......@@ -65,16 +65,16 @@ class TestSecurityGroups(object):
get_group_members.return_value = ("aa:bb:cc:dd:ee:ff", ["aa:bb:cc:dd:ee:ff"])
etcd_client = mock.MagicMock()
security_groups.delete_mac_from_group(etcd_client, sec_group, "11:22:33:44:55:66")
assert not etcd_client.testandset.called
assert not etcd_client.test_and_set.called
@mock.patch('nova_router.security_groups.get_group_members')
def test_delete_mac_from_group_empty(self, get_group_members):
get_group_members.return_value = ("aa:bb:cc:dd:ee:ff", ["aa:bb:cc:dd:ee:ff"])
etcd_client = mock.MagicMock()
security_groups.delete_mac_from_group(etcd_client, sec_group, "aa:bb:cc:dd:ee:ff")
etcd_client.testandset.assert_called_with(self.members_key,
"aa:bb:cc:dd:ee:ff",
"\n")
etcd_client.test_and_set.assert_called_with(self.members_key,
"\n",
"aa:bb:cc:dd:ee:ff")
@mock.patch('nova_router.security_groups.get_group_members')
def test_delete_mac_from_group_not_empty(self, get_group_members):
......@@ -82,9 +82,9 @@ class TestSecurityGroups(object):
["aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"])
etcd_client = mock.MagicMock()
security_groups.delete_mac_from_group(etcd_client, sec_group, "aa:bb:cc:dd:ee:ff")
etcd_client.testandset.assert_called_with(self.members_key,
"aa:bb:cc:dd:ee:ff\n11:22:33:44:55:66",
"11:22:33:44:55:66")
etcd_client.test_and_set.assert_called_with(self.members_key,
"11:22:33:44:55:66",
"aa:bb:cc:dd:ee:ff\n11:22:33:44:55:66")
@mock.patch("nova_router.security_groups.get_security_groups")
def test_get_groups_from_mac_no_groups(self, groups):
......@@ -126,14 +126,14 @@ class TestGetGroupMembers(object):
assert members == ["aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"]
def test_no_key(self):
self.etcd.get = mock.MagicMock(side_effect=etcd.EtcdError(100, "Key Not Found"))
self.etcd.get = mock.MagicMock(side_effect=KeyError("Key Not Found"))
raw, members = security_groups.get_group_members(self.etcd, sec_group)
assert raw == ""
assert members == []
def test_raises_other_errors(self):
self.etcd.get = mock.MagicMock(side_effect=etcd.EtcdError(101, "Ugle Key Not Found"))
with raises(etcd.EtcdError):
self.etcd.get = mock.MagicMock(side_effect=etcd.EtcdException("Ugle Key Not Found"))
with raises(etcd.EtcdException):
raw, members = security_groups.get_group_members(self.etcd, sec_group)
......
......@@ -5,7 +5,7 @@ from setuptools import setup, find_packages
here = os.path.abspath(os.path.dirname(__file__))
requires = [
'etcd-py>=0.0.9',
'python-etcd',
'requests',
'daemonize',
'pytest',
......@@ -26,7 +26,6 @@ setup(name='router-services',
install_requires=requires,
tests_require=requires,
dependency_links=[
'git+https://github.com/sigmunau/etcd-py.git#egg=etcd-py-0.0.9',
'git+https://scm.uninett.no/nova/scvmm_client_py.git#egg=scvmm-client-py-0.4',
],
)
......@@ -31,7 +31,7 @@ def parse_args():
def dump_subtree(etcd, tree, dump):
for entry in etcd.list(tree):
for entry in etcd.read(tree).children:
if entry.dir:
dump_subtree(etcd, entry.key, dump)
else:
......@@ -40,8 +40,8 @@ def dump_subtree(etcd, tree, dump):
def main():
args = parse_args()
etcd_client = etcd.Etcd(args.etcd_host, 4001, ssl_cert=args.cert,
ssl_key=args.key, verify=args.cacert)
etcd_client = etcd.Client(host=args.etcd_host, cert=(args.cert,args.key),
ca_cert=args.cacert, protocol='https')
data = {}
dump_subtree(etcd_client, args.subtree, data)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment