Add a way to filter nodes by user agent (exclude_uagent_string) (#13)

 Add a way to filter nodes by user agent (exclude_uagent_string)

Author: sickpig

conf/export.conf.default

@@ -14,3 +14,6 @@ debug = False
 
 # Relative path to directory containing timestamp-prefixed JSON export files
 export_dir = data/export/e3e1f3e8
+
+# filter by user_agent
+exclude_uagent_string =

export.py

@@ -33,6 +33,8 @@
 import os
 import sys
 import time
+import re
+
 from binascii import hexlify, unhexlify
 from ConfigParser import ConfigParser
 
@@ -48,6 +50,10 @@ def get_row(node):
     """
     # address, port, version, user_agent, timestamp, services
     node = eval(node)
+    uagent = node[3]
+    p = re.compile(CONF['exclude_uagent_string'])
+    if (p.search(uagent) is not None):
+        return ''
     address = node[0]
     port = node[1]
     services = node[-1]
@@ -83,7 +89,8 @@ def export_nodes(nodes, timestamp):
     start = time.time()
     for node in nodes:
         row = get_row(node)
-        rows.append(row)
+        if (row != ''):
+            rows.append(row)
     end = time.time()
     elapsed = end - start
     logging.info("Elapsed: %d", elapsed)
@@ -106,6 +113,7 @@ def init_conf(argv):
     CONF['db'] = conf.getint('export', 'db')
     CONF['debug'] = conf.getboolean('export', 'debug')
     CONF['export_dir'] = conf.get('export', 'export_dir')
+    CONF['exclude_uagent_string'] = conf.get('export', 'exclude_uagent_string')
     if not os.path.exists(CONF['export_dir']):
         os.makedirs(CONF['export_dir'])