|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +# ------------------------------------------------------------------------------- |
| 3 | +# Name: sfp_deepinfo |
| 4 | +# Purpose: Query Deepinfo using their API |
| 5 | +# |
| 6 | +# Author: Ceylan Bozogullarindan |
| 7 | +# |
| 8 | +# Created: 16/04:/2022 |
| 9 | +# Copyright: (c) Deepinfo 2023 |
| 10 | +# Licence: GPL |
| 11 | +# ------------------------------------------------------------------------------- |
| 12 | + |
| 13 | +import json |
| 14 | +import time |
| 15 | +from spiderfoot import SpiderFootEvent, SpiderFootPlugin |
| 16 | + |
| 17 | +class sfp_deepinfo(SpiderFootPlugin): |
| 18 | + meta = { |
| 19 | + 'name': "Deepinfo", |
| 20 | + 'summary': "Obtain Passive DNS and other information from Deepinfo", |
| 21 | + 'flags': ["apikey"], |
| 22 | + 'useCases': ["Investigate", "Passive"], |
| 23 | + 'categories': ["Search Engines"], |
| 24 | + 'dataSource': { |
| 25 | + 'website': "https://deepinfo.com/", |
| 26 | + 'model': "COMMERCIAL_ONLY", |
| 27 | + 'references': [ |
| 28 | + "https://docs.deepinfo.com/docs/getting-started" |
| 29 | + ], |
| 30 | + 'apiKeyInstructions': [ |
| 31 | + "Visit https://deepinfo.info/request-demo", |
| 32 | + "Request a demo account", |
| 33 | + "Navigate to https://platform.deepinfo.com/app/settings/organization/api-keys", |
| 34 | + "The API key is listed under 'API Keys'" |
| 35 | + ], |
| 36 | + 'favIcon': "https://ik.imagekit.io/deepinfo/test/favicon/favicon-96x96.png", |
| 37 | + 'logo': "https://ik.imagekit.io/deepinfo/test/favicon/favicon-96x96.png", |
| 38 | + 'description': "Deepinfo provides relevant data and insights that empower " |
| 39 | + "cybersecurity professionals by providing access to an extensive database and reliable indicators." |
| 40 | + "Deepinfo has the data you need to understand what's going on on the Internet, we are dealing with " |
| 41 | + "terabytes of data, hundreds of data sources, billions of lines of raw data.", |
| 42 | + } |
| 43 | + } |
| 44 | + |
| 45 | + # Default options |
| 46 | + opts = { |
| 47 | + "api_key": "", |
| 48 | + } |
| 49 | + |
| 50 | + # Option descriptions |
| 51 | + optdescs = { |
| 52 | + "api_key": "Deepinfo API key.", |
| 53 | + } |
| 54 | + |
| 55 | + # Be sure to completely clear any class variables in setup() |
| 56 | + # or you run the risk of data persisting between scan runs. |
| 57 | + |
| 58 | + results = None |
| 59 | + errorState = False |
| 60 | + |
| 61 | + def setup(self, sfc, userOpts=dict()): |
| 62 | + self.sf = sfc |
| 63 | + self.results = self.tempStorage() |
| 64 | + |
| 65 | + # Clear / reset any other class member variables here |
| 66 | + # or you risk them persisting between threads. |
| 67 | + |
| 68 | + for opt in list(userOpts.keys()): |
| 69 | + self.opts[opt] = userOpts[opt] |
| 70 | + |
| 71 | + # What events is this module interested in for input |
| 72 | + def watchedEvents(self): |
| 73 | + return ["DOMAIN_NAME"] |
| 74 | + |
| 75 | + # What events this module produces |
| 76 | + def producedEvents(self): |
| 77 | + return ["DOMAIN_NAME", |
| 78 | + "INTERNET_NAME" |
| 79 | + ] |
| 80 | + |
| 81 | + # Search Deepinfo |
| 82 | + def query(self, qry, page=1, accum=None): |
| 83 | + url = f"https://api.deepinfo.com/v1/discovery/subdomain-finder?domain={qry}&page={page}" |
| 84 | + request = None |
| 85 | + headers = {"apikey": self.opts['api_key']} |
| 86 | + res = self.sf.fetchUrl(url, |
| 87 | + useragent="SpiderFoot", headers=headers, |
| 88 | + postData=request) |
| 89 | + |
| 90 | + if res['code'] not in ["200"]: |
| 91 | + self.error("Deepinfo API key seems to have been rejected or you have exceeded usage limits for the month.") |
| 92 | + self.errorState = True |
| 93 | + return None |
| 94 | + |
| 95 | + if res['content'] is None: |
| 96 | + self.info("No Deepinfo info found for " + qry) |
| 97 | + return None |
| 98 | + |
| 99 | + try: |
| 100 | + info = json.loads(res['content']) |
| 101 | + self.info("result_count {0}, page {1}".format(info.get("result_count"), page)) |
| 102 | + if info.get("result_count", 0) > 100: |
| 103 | + domains = [item.get("punycode", "") for item in info.get("results", [])] |
| 104 | + if len(domains) >= 100: |
| 105 | + # Avoid throttling |
| 106 | + time.sleep(1) |
| 107 | + if accum: |
| 108 | + accum.extend(domains) |
| 109 | + else: |
| 110 | + accum = domains |
| 111 | + return self.query(qry, page + 1, accum) |
| 112 | + else: |
| 113 | + # We are at the last page |
| 114 | + accum.extend(domains) |
| 115 | + return accum |
| 116 | + else: |
| 117 | + return info.get('results', []) |
| 118 | + except Exception as e: |
| 119 | + self.error("Error processing JSON response from Deepinfo: " + str(e)) |
| 120 | + return None |
| 121 | + |
| 122 | + # Handle events sent to this module |
| 123 | + def handleEvent(self, event): |
| 124 | + eventName = event.eventType |
| 125 | + srcModuleName = event.module |
| 126 | + eventData = event.data |
| 127 | + |
| 128 | + if self.errorState: |
| 129 | + return |
| 130 | + |
| 131 | + self.debug(f"Received event, {eventName}, from {srcModuleName}") |
| 132 | + |
| 133 | + if self.opts['api_key'] == "": |
| 134 | + self.error("You enabled sfp_deepinfo but did not set an API uid/secret!") |
| 135 | + self.errorState = True |
| 136 | + return |
| 137 | + |
| 138 | + # Don't look up stuff twice |
| 139 | + if eventData in self.results: |
| 140 | + self.debug(f"Skipping {eventData}, already checked.") |
| 141 | + return |
| 142 | + |
| 143 | + self.results[eventData] = True |
| 144 | + |
| 145 | + if eventName in ["DOMAIN_NAME"]: |
| 146 | + domain = eventData |
| 147 | + rec = self.query(domain) |
| 148 | + myres = list() |
| 149 | + if rec is not None: |
| 150 | + for h in rec: |
| 151 | + if h == "": |
| 152 | + continue |
| 153 | + if h.lower() not in myres: |
| 154 | + myres.append(h.lower()) |
| 155 | + else: |
| 156 | + continue |
| 157 | + e = SpiderFootEvent("INTERNET_NAME", h, |
| 158 | + self.__name__, event) |
| 159 | + self.notifyListeners(e) |
0 commit comments