Improved JSON parsing and error handling for tool modules

modules/sfp_tool_nuclei.py

@@ -155,7 +155,7 @@ def handleEvent(self, event):
             args = [
                 exe,
                 "-silent",
-                "-json",
+                "-jsonl",
                 "-concurrency",
                 "100",
                 "-retries",
@@ -189,53 +189,99 @@ def handleEvent(self, event):
         if not content:
             return
 
-        try:
-            for line in content.split("\n"):
-                if not line:
-                    continue
-
+        for line in content.split("\n"):
+            if not line.strip():
+                continue
+
+            if not line.strip().startswith('{'):
+                continue
+
+            try:
                 data = json.loads(line)
+
+                if 'matched-at' not in data or 'info' not in data:
+                    continue
+
                 srcevent = event
-                host = data['matched-at'].split(":")[0]
-                if host != eventData:
+                matched_url = data.get('matched-at', '')
+                if matched_url:
+                    if matched_url.startswith('http'):
+                        from urllib.parse import urlparse
+                        parsed = urlparse(matched_url)
+                        host = parsed.hostname
+                    else:
+                        host = data.get('host', eventData)
+                else:
+                    host = data.get('host', eventData)
+
+                if host and host != eventData:
                     if self.sf.validIP(host):
                         srctype = "IP_ADDRESS"
                     else:
                         srctype = "INTERNET_NAME"
                     srcevent = SpiderFootEvent(srctype, host, self.__name__, event)
                     self.notifyListeners(srcevent)
 
-                matches = re.findall(r"CVE-\d{4}-\d{4,7}", line)
-                if matches:
-                    for cve in matches:
-                        etype, cvetext = self.sf.cveInfo(cve)
-                        e = SpiderFootEvent(
-                            etype, cvetext, self.__name__, srcevent
-                        )
-                        self.notifyListeners(e)
+
+                template_info = data.get('info', {})
+                cve_list = []
+
+                classification = template_info.get('classification', {})
+                if classification.get('cve-id'):
+                    cve_id = classification.get('cve-id')
+                    if isinstance(cve_id, list):
+                        cve_list.extend(cve_id)
+                    elif isinstance(cve_id, str):
+                        cve_list.append(cve_id)
+
+                template_id = data.get('template-id', '')
+                if 'CVE-' in template_id:
+                    matches = re.findall(r"CVE-\d{4}-\d{4,7}", template_id)
+                    cve_list.extend(matches)
+
+                if cve_list:
+                    for cve in cve_list:
+                        if cve and cve != 'null':
+                            etype, cvetext = self.sf.cveInfo(cve)
+                            e = SpiderFootEvent(etype, cvetext, self.__name__, srcevent)
+                            self.notifyListeners(e)
                 else:
-                    if "matcher-name" in data:
+                    severity = template_info.get('severity', 'info').lower()
+
+                    if severity == "info":
+                        etype = "WEBSERVER_TECHNOLOGY"
+                    else:
                         etype = "VULNERABILITY_GENERAL"
-                        if data['info']['severity'] == "info":
-                            etype = "WEBSERVER_TECHNOLOGY"
-
-                        datatext = f"Template: {data['info']['name']}({data['template-id']})\n"
-                        datatext += f"Matcher: {data['matcher-name']}\n"
-                        datatext += f"Matched at: {data['matched-at']}\n"
-                        if data['info'].get('reference'):
-                            datatext += f"Reference: <SFURL>{data['info']['reference'][0]}</SFURL>"
-
-                        evt = SpiderFootEvent(
-                            etype,
-                            datatext,
-                            self.__name__,
-                            srcevent,
-                        )
-                        self.notifyListeners(evt)
-        except (KeyError, ValueError) as e:
-            self.error(f"Couldn't parse the JSON output of Nuclei: {e}")
-            self.error(f"Nuclei content: {content}")
-            return
-
+
+                    datatext = f"Template: {template_info.get('name', 'Unknown')} ({data.get('template-id', 'Unknown')})\n"
+
+                    if data.get('matcher-name'):
+                        datatext += f"Matcher: {data.get('matcher-name')}\n"
+
+                    if data.get('matched-at'):
+                        datatext += f"Matched at: {data.get('matched-at')}\n"
+
+                    if severity != 'info':
+                        datatext += f"Severity: {severity}\n"
+
+                    if template_info.get('description'):
+                        datatext += f"Description: {template_info.get('description')}\n"
+
+                    references = template_info.get('reference', [])
+                    if references:
+                        if isinstance(references, list) and len(references) > 0:
+                            datatext += f"Reference: <SFURL>{references[0]}</SFURL>"
+                        elif isinstance(references, str):
+                            datatext += f"Reference: <SFURL>{references}</SFURL>"
+
+                    evt = SpiderFootEvent(etype, datatext, self.__name__, srcevent)
+                    self.notifyListeners(evt)
+
+            except json.JSONDecodeError as e:
+                self.debug(f"Skipping non-JSON line: {line}")
+                continue
+            except KeyError as e:
+                self.debug(f"Missing expected key in Nuclei output: {e}")
+                continue
 
 # End of sfp_tool_nuclei class

modules/sfp_tool_wafw00f.py

@@ -13,7 +13,8 @@
 
 import json
 import os.path
-from subprocess import PIPE, Popen, TimeoutExpired
+import tempfile
+from subprocess import PIPE, Popen
 
 from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers
 
@@ -97,23 +98,20 @@ def handleEvent(self, event):
             self.error("Invalid input, refusing to run.")
             return
 
+        output_file = tempfile.mktemp(suffix=".json")
+
         args = [
             self.opts['python_path'],
             exe,
             '-a',
-            '-o-',
-            '-f',
-            'json',
+            '-o',
+            output_file,
             url
         ]
+
         try:
             p = Popen(args, stdout=PIPE, stderr=PIPE)
-            stdout, stderr = p.communicate(input=None, timeout=300)
-        except TimeoutExpired:
-            p.kill()
-            stdout, stderr = p.communicate()
-            self.debug(f"Timed out waiting for wafw00f to finish on {eventData}")
-            return
+            stdout, stderr = p.communicate(input=None)
         except Exception as e:
             self.error(f"Unable to run wafw00f: {e}")
             return
@@ -122,15 +120,15 @@ def handleEvent(self, event):
             self.error(f"Unable to read wafw00f output\nstderr: {stderr}\nstdout: {stdout}")
             return
 
-        if not stdout:
-            self.debug(f"wafw00f returned no output for {eventData}")
-            return
-
         try:
-            result_json = json.loads(stdout)
+            with open(output_file, "r") as f:
+                result_json = json.load(f)
         except Exception as e:
-            self.error(f"Could not parse wafw00f output as JSON: {e}\nstdout: {stdout}")
+            self.error(f"Could not parse wafw00f output as JSON from file: {e}")
             return
+        finally:
+            if os.path.exists(output_file):
+                os.remove(output_file)
 
         if not result_json:
             self.debug(f"wafw00f returned no output for {eventData}")

modules/sfp_tool_whatweb.py

@@ -8,12 +8,12 @@
 #
 # Created:     2019-08-31
 # Copyright:   (c) bcoles 2019
-# Licence:     MIT
+# Licence:     GPL
 # -------------------------------------------------------------------------------
 
 import json
 import os.path
-from subprocess import PIPE, Popen, TimeoutExpired
+from subprocess import PIPE, Popen
 
 from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers
 
@@ -121,40 +121,54 @@ def handleEvent(self, event):
             self.opts['ruby_path'],
             exe,
             "--quiet",
+            "--no-errors",
             "--aggression=" + str(aggression),
             "--log-json=/dev/stdout",
             "--user-agent=Mozilla/5.0",
             "--follow-redirect=never",
             eventData
         ]
         try:
-            p = Popen(args, stdout=PIPE, stderr=PIPE, timeout=300)
+            p = Popen(args, stdout=PIPE, stderr=PIPE)
             stdout, stderr = p.communicate(input=None)
-        except TimeoutExpired:
-            p.kill()
-            stdout, stderr = p.communicate()
-            self.debug(f"Timed out waiting for WhatWeb to finish against {eventData}")
-            return
         except Exception as e:
             self.error(f"Unable to run WhatWeb: {e}")
             return
 
         if p.returncode != 0:
             self.error("Unable to read WhatWeb output.")
-            self.debug("Error running WhatWeb: " + stderr + ", " + stdout)
+            self.debug("Error running WhatWeb: " + stderr.decode('utf-8', errors='ignore') + ", " + stdout.decode('utf-8', errors='ignore'))
             return
 
         if not stdout:
             self.debug(f"WhatWeb returned no output for {eventData}")
             return
 
+        # Parse JSON output - WhatWeb returns one JSON object per line
+        result_json = []
         try:
-            result_json = json.loads(stdout)
+            stdout_str = stdout.decode('utf-8', errors='ignore').strip()
+
+            # Debug: mostrar o que foi retornado
+            self.debug(f"WhatWeb output (first 500 chars): {stdout_str[:500]}")
+
+            # WhatWeb retorna múltiplas linhas de JSON, uma por linha
+            for line in stdout_str.split('\n'):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    result_json.append(json.loads(line))
+                except json.JSONDecodeError as e:
+                    self.debug(f"Failed to parse JSON line: {line[:100]} - Error: {e}")
+                    continue
+
         except Exception as e:
             self.error(f"Couldn't parse the JSON output of WhatWeb: {e}")
             return
 
         if len(result_json) == 0:
+            self.debug(f"No valid JSON results found for {eventData}")
             return
 
         blacklist = [
@@ -173,16 +187,20 @@ def handleEvent(self, event):
                 continue
 
             if plugin_matches.get('HTTPServer'):
-                for w in plugin_matches.get('HTTPServer').get('string'):
-                    evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event)
-                    self.notifyListeners(evt)
-                    found = True
+                http_server_data = plugin_matches.get('HTTPServer')
+                if isinstance(http_server_data, dict) and http_server_data.get('string'):
+                    for w in http_server_data.get('string'):
+                        evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event)
+                        self.notifyListeners(evt)
+                        found = True
 
             if plugin_matches.get('X-Powered-By'):
-                for w in plugin_matches.get('X-Powered-By').get('string'):
-                    evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event)
-                    self.notifyListeners(evt)
-                    found = True
+                x_powered_data = plugin_matches.get('X-Powered-By')
+                if isinstance(x_powered_data, dict) and x_powered_data.get('string'):
+                    for w in x_powered_data.get('string'):
+                        evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event)
+                        self.notifyListeners(evt)
+                        found = True
 
             for plugin in plugin_matches:
                 if plugin in blacklist:
@@ -195,4 +213,4 @@ def handleEvent(self, event):
             evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event)
             self.notifyListeners(evt)
 
-# End of sfp_tool_whatweb class
+# End of sfp_tool_whatweb class.