diff --git a/modules/sfp_tool_nuclei.py b/modules/sfp_tool_nuclei.py
index 58c12b0477..fac7aca2bb 100644
--- a/modules/sfp_tool_nuclei.py
+++ b/modules/sfp_tool_nuclei.py
@@ -155,7 +155,7 @@ def handleEvent(self, event):
args = [
exe,
"-silent",
- "-json",
+ "-jsonl",
"-concurrency",
"100",
"-retries",
@@ -189,15 +189,32 @@ def handleEvent(self, event):
if not content:
return
- try:
- for line in content.split("\n"):
- if not line:
- continue
-
+ for line in content.split("\n"):
+ if not line.strip():
+ continue
+
+ if not line.strip().startswith('{'):
+ continue
+
+ try:
data = json.loads(line)
+
+ if 'matched-at' not in data or 'info' not in data:
+ continue
+
srcevent = event
- host = data['matched-at'].split(":")[0]
- if host != eventData:
+ matched_url = data.get('matched-at', '')
+ if matched_url:
+ if matched_url.startswith('http'):
+ from urllib.parse import urlparse
+ parsed = urlparse(matched_url)
+ host = parsed.hostname
+ else:
+ host = data.get('host', eventData)
+ else:
+ host = data.get('host', eventData)
+
+ if host and host != eventData:
if self.sf.validIP(host):
srctype = "IP_ADDRESS"
else:
@@ -205,37 +222,66 @@ def handleEvent(self, event):
srcevent = SpiderFootEvent(srctype, host, self.__name__, event)
self.notifyListeners(srcevent)
- matches = re.findall(r"CVE-\d{4}-\d{4,7}", line)
- if matches:
- for cve in matches:
- etype, cvetext = self.sf.cveInfo(cve)
- e = SpiderFootEvent(
- etype, cvetext, self.__name__, srcevent
- )
- self.notifyListeners(e)
+
+ template_info = data.get('info', {})
+ cve_list = []
+
+ classification = template_info.get('classification', {})
+ if classification.get('cve-id'):
+ cve_id = classification.get('cve-id')
+ if isinstance(cve_id, list):
+ cve_list.extend(cve_id)
+ elif isinstance(cve_id, str):
+ cve_list.append(cve_id)
+
+ template_id = data.get('template-id', '')
+ if 'CVE-' in template_id:
+ matches = re.findall(r"CVE-\d{4}-\d{4,7}", template_id)
+ cve_list.extend(matches)
+
+ if cve_list:
+ for cve in cve_list:
+ if cve and cve != 'null':
+ etype, cvetext = self.sf.cveInfo(cve)
+ e = SpiderFootEvent(etype, cvetext, self.__name__, srcevent)
+ self.notifyListeners(e)
else:
- if "matcher-name" in data:
+ severity = template_info.get('severity', 'info').lower()
+
+ if severity == "info":
+ etype = "WEBSERVER_TECHNOLOGY"
+ else:
etype = "VULNERABILITY_GENERAL"
- if data['info']['severity'] == "info":
- etype = "WEBSERVER_TECHNOLOGY"
-
- datatext = f"Template: {data['info']['name']}({data['template-id']})\n"
- datatext += f"Matcher: {data['matcher-name']}\n"
- datatext += f"Matched at: {data['matched-at']}\n"
- if data['info'].get('reference'):
- datatext += f"Reference: {data['info']['reference'][0]}"
-
- evt = SpiderFootEvent(
- etype,
- datatext,
- self.__name__,
- srcevent,
- )
- self.notifyListeners(evt)
- except (KeyError, ValueError) as e:
- self.error(f"Couldn't parse the JSON output of Nuclei: {e}")
- self.error(f"Nuclei content: {content}")
- return
-
+
+ datatext = f"Template: {template_info.get('name', 'Unknown')} ({data.get('template-id', 'Unknown')})\n"
+
+ if data.get('matcher-name'):
+ datatext += f"Matcher: {data.get('matcher-name')}\n"
+
+ if data.get('matched-at'):
+ datatext += f"Matched at: {data.get('matched-at')}\n"
+
+ if severity != 'info':
+ datatext += f"Severity: {severity}\n"
+
+ if template_info.get('description'):
+ datatext += f"Description: {template_info.get('description')}\n"
+
+ references = template_info.get('reference', [])
+ if references:
+ if isinstance(references, list) and len(references) > 0:
+ datatext += f"Reference: {references[0]}"
+ elif isinstance(references, str):
+ datatext += f"Reference: {references}"
+
+ evt = SpiderFootEvent(etype, datatext, self.__name__, srcevent)
+ self.notifyListeners(evt)
+
+ except json.JSONDecodeError as e:
+ self.debug(f"Skipping non-JSON line: {line}")
+ continue
+ except KeyError as e:
+ self.debug(f"Missing expected key in Nuclei output: {e}")
+ continue
# End of sfp_tool_nuclei class
diff --git a/modules/sfp_tool_wafw00f.py b/modules/sfp_tool_wafw00f.py
index 8e508af51d..874b81a79f 100644
--- a/modules/sfp_tool_wafw00f.py
+++ b/modules/sfp_tool_wafw00f.py
@@ -13,7 +13,8 @@
import json
import os.path
-from subprocess import PIPE, Popen, TimeoutExpired
+import tempfile
+from subprocess import PIPE, Popen
from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers
@@ -97,23 +98,20 @@ def handleEvent(self, event):
self.error("Invalid input, refusing to run.")
return
+ output_file = tempfile.mktemp(suffix=".json")
+
args = [
self.opts['python_path'],
exe,
'-a',
- '-o-',
- '-f',
- 'json',
+ '-o',
+ output_file,
url
]
+
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
- stdout, stderr = p.communicate(input=None, timeout=300)
- except TimeoutExpired:
- p.kill()
- stdout, stderr = p.communicate()
- self.debug(f"Timed out waiting for wafw00f to finish on {eventData}")
- return
+ stdout, stderr = p.communicate(input=None)
except Exception as e:
self.error(f"Unable to run wafw00f: {e}")
return
@@ -122,15 +120,15 @@ def handleEvent(self, event):
self.error(f"Unable to read wafw00f output\nstderr: {stderr}\nstdout: {stdout}")
return
- if not stdout:
- self.debug(f"wafw00f returned no output for {eventData}")
- return
-
try:
- result_json = json.loads(stdout)
+ with open(output_file, "r") as f:
+ result_json = json.load(f)
except Exception as e:
- self.error(f"Could not parse wafw00f output as JSON: {e}\nstdout: {stdout}")
+ self.error(f"Could not parse wafw00f output as JSON from file: {e}")
return
+ finally:
+ if os.path.exists(output_file):
+ os.remove(output_file)
if not result_json:
self.debug(f"wafw00f returned no output for {eventData}")
diff --git a/modules/sfp_tool_whatweb.py b/modules/sfp_tool_whatweb.py
index a226c095f2..55d2c1e390 100644
--- a/modules/sfp_tool_whatweb.py
+++ b/modules/sfp_tool_whatweb.py
@@ -8,12 +8,12 @@
#
# Created: 2019-08-31
# Copyright: (c) bcoles 2019
-# Licence: MIT
+# Licence: GPL
# -------------------------------------------------------------------------------
import json
import os.path
-from subprocess import PIPE, Popen, TimeoutExpired
+from subprocess import PIPE, Popen
from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers
@@ -121,6 +121,7 @@ def handleEvent(self, event):
self.opts['ruby_path'],
exe,
"--quiet",
+ "--no-errors",
"--aggression=" + str(aggression),
"--log-json=/dev/stdout",
"--user-agent=Mozilla/5.0",
@@ -128,33 +129,46 @@ def handleEvent(self, event):
eventData
]
try:
- p = Popen(args, stdout=PIPE, stderr=PIPE, timeout=300)
+ p = Popen(args, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
- except TimeoutExpired:
- p.kill()
- stdout, stderr = p.communicate()
- self.debug(f"Timed out waiting for WhatWeb to finish against {eventData}")
- return
except Exception as e:
self.error(f"Unable to run WhatWeb: {e}")
return
if p.returncode != 0:
self.error("Unable to read WhatWeb output.")
- self.debug("Error running WhatWeb: " + stderr + ", " + stdout)
+ self.debug("Error running WhatWeb: " + stderr.decode('utf-8', errors='ignore') + ", " + stdout.decode('utf-8', errors='ignore'))
return
if not stdout:
self.debug(f"WhatWeb returned no output for {eventData}")
return
+ # Parse JSON output - WhatWeb returns one JSON object per line
+ result_json = []
try:
- result_json = json.loads(stdout)
+ stdout_str = stdout.decode('utf-8', errors='ignore').strip()
+
+ # Debug: mostrar o que foi retornado
+ self.debug(f"WhatWeb output (first 500 chars): {stdout_str[:500]}")
+
+ # WhatWeb retorna múltiplas linhas de JSON, uma por linha
+ for line in stdout_str.split('\n'):
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ result_json.append(json.loads(line))
+ except json.JSONDecodeError as e:
+ self.debug(f"Failed to parse JSON line: {line[:100]} - Error: {e}")
+ continue
+
except Exception as e:
self.error(f"Couldn't parse the JSON output of WhatWeb: {e}")
return
if len(result_json) == 0:
+ self.debug(f"No valid JSON results found for {eventData}")
return
blacklist = [
@@ -173,16 +187,20 @@ def handleEvent(self, event):
continue
if plugin_matches.get('HTTPServer'):
- for w in plugin_matches.get('HTTPServer').get('string'):
- evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event)
- self.notifyListeners(evt)
- found = True
+ http_server_data = plugin_matches.get('HTTPServer')
+ if isinstance(http_server_data, dict) and http_server_data.get('string'):
+ for w in http_server_data.get('string'):
+ evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event)
+ self.notifyListeners(evt)
+ found = True
if plugin_matches.get('X-Powered-By'):
- for w in plugin_matches.get('X-Powered-By').get('string'):
- evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event)
- self.notifyListeners(evt)
- found = True
+ x_powered_data = plugin_matches.get('X-Powered-By')
+ if isinstance(x_powered_data, dict) and x_powered_data.get('string'):
+ for w in x_powered_data.get('string'):
+ evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event)
+ self.notifyListeners(evt)
+ found = True
for plugin in plugin_matches:
if plugin in blacklist:
@@ -195,4 +213,4 @@ def handleEvent(self, event):
evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event)
self.notifyListeners(evt)
-# End of sfp_tool_whatweb class
+# End of sfp_tool_whatweb class.