Replace two maps and a union with a flatMap. Seems to be faster

3e8434ea · Sigmund Augdal · 0ac1f5ac · 3e8434ea
Commit 3e8434ea authored Jul 07, 2014 by Sigmund Augdal
Hide whitespace changes
Inline Side-by-side

Showing with 5 additions and 3 deletions

pythonApp/netflowAlgs.py pythonApp/netflowAlgs.py +5 -3

No files found.
--- a/pythonApp/netflowAlgs.py
+++ b/pythonApp/netflowAlgs.py
@@ -31,7 +31,9 @@ def ports_count_by_ip3(csv):


 def ports_count_by_ip(csv):
-    srcs = csv.map(lambda x: ((x[DEST_PORT], x[SRC_IP]), 1))
-    dsts = csv.map(lambda x: ((x[DEST_PORT], x[DEST_IP]), 1))
-    ips = srcs.union(dsts).reduceByKey(add)
+#    srcs = csv.map(lambda x: ((x[DEST_PORT], x[SRC_IP]), 1))
+#    dsts = csv.map(lambda x: ((x[DEST_PORT], x[DEST_IP]), 1))
+#    ips = srcs.union(dsts).reduceByKey(add)
+    ips = csv.flatMap(lambda x: (((x[DEST_PORT], x[DEST_IP]), 1),
+                                 ((x[DEST_PORT], x[SRC_IP]), 1))).reduceByKey(add)
    return ips.map(lambda x: (x[1], x[0])).sortByKey(False).take(20)