Merge pull request #293 from 13ph03nix/master

13ph03nix · web-flow · commit 436904403077 · 2022-07-06T14:29:27.000-07:00
Some improvements &amp; Bug fixes
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -343,3 +343,10 @@ Cross-platform shell code generation
 * target url support cidr, user can use -p provide additional ports
 * support local mode, local mode do not need any targets, e.g. LPE
 * bug fixes
+
+# version 1.9.6
+----------------
+* support -o parameter, save the result in json lines format
+* optimize timeout for cyberspace search engine plugins
+* optimize the handling of the url protocol
+* docs update
diff --git a/Dockerfile b/Dockerfile
@@ -31,7 +31,7 @@ RUN sh -c "$(wget -O- https://raw.githubusercontent.com/13ph03nix/zsh-in-docker/
     && sudo apt-get clean -y \
     && sudo rm -rf /var/lib/apt/lists/*
 
-RUN sudo pip3 install --upgrade pip && sudo pip3 install --upgrade pocsuite3==1.9.5
+RUN sudo pip3 install --upgrade pip && sudo pip3 install --upgrade pocsuite3==1.9.6
 
 WORKDIR /home/pocsuite3
 CMD ["zsh"]
diff --git a/README.md b/README.md
@@ -130,7 +130,7 @@ cli mode
 	# run poc with shell mode
 	pocsuite -u http://example.com -r example.py -v 2 --shell
 
-	# search for the target of redis service from ZoomEye and perform batch detection of vulnerabilities. The thread is set to 20
+	# search for the target of redis service from ZoomEye and perform batch detection of vulnerabilities. The threads is set to 20
 	pocsuite -r redis.py --dork service:redis --threads 20
 
 	# load all poc in the poc directory and save the result as html
diff --git a/docs/USAGE.md b/docs/USAGE.md
@@ -49,7 +49,7 @@ Request:
   --host HOST           HTTP Host header value
   --referer REFERER     HTTP Referer header value
   --user-agent AGENT    HTTP User-Agent header value (default random)
-  --proxy PROXY         Use a proxy to connect to the target URL
+  --proxy PROXY         Use a proxy to connect to the target URL (protocol://host:port)
   --proxy-cred PROXY_CRED
                         Proxy authentication credentials (name:password)
   --timeout TIMEOUT     Seconds to wait before timeout connection (default 10)
@@ -118,6 +118,8 @@ Modules:
 Optimization:
   Optimization options
 
+  -o OUTPUT_PATH, --output OUTPUT_PATH
+                        Output file to write (JSON Lines format)
   --plugins PLUGINS     Load plugins to execute
   --pocs-path POCS_PATH
                         User defined poc scripts path
@@ -208,7 +210,7 @@ $ pocsuite --dork 'port:6379' --vul-keyword 'redis' --max-page 2
  Search libssh server  with  `libssh` keyword.
 
  ```
- pocsuite -r pocs/libssh_auth_bypass.py --dork-shodan libssh --thread 10
+ pocsuite -r pocs/libssh_auth_bypass.py --dork-shodan libssh --threads 10
  ```
 
 **--dork-fofa DORK**
@@ -219,7 +221,7 @@ $ pocsuite --dork 'port:6379' --vul-keyword 'redis' --max-page 2
 
 
  ```
- $ pocsuite -r pocs/check_http_status.py --dork-fofa 'body="thinkphp"' --search-type web --thread 10
+ $ pocsuite -r pocs/check_http_status.py --dork-fofa 'body="thinkphp"' --search-type web --threads 10
  ```
 
 **--dork-quake DORK**
@@ -230,7 +232,7 @@ $ pocsuite --dork 'port:6379' --vul-keyword 'redis' --max-page 2
 
 
  ```
- $ pocsuite -r pocs/check_http_status.py --dork-quake 'app:"ThinkPHP"' --thread 10
+ $ pocsuite -r pocs/check_http_status.py --dork-quake 'app:"ThinkPHP"' --threads 10
  ```
 
 **--dork-b64**
@@ -271,7 +273,7 @@ cli mode
 	# run poc with shell mode
 	pocsuite -u http://example.com -r example.py -v 2 --shell
 
-	# search for the target of redis service from ZoomEye and perform batch detection of vulnerabilities. The thread is set to 20
+	# search for the target of redis service from ZoomEye and perform batch detection of vulnerabilities. The threads is set to 20
 	pocsuite -r redis.py --dork service:redis --threads 20
 
 	# load all poc in the poc directory and save the result as html
diff --git a/manpages/poc-console.1 b/manpages/poc-console.1
@@ -1,6 +1,6 @@
-.TH POC-CONSOLE "1" "May 2022" "Manual page for poc-console"
+.TH POC-CONSOLE "1" "July 2022" "Manual page for poc-console"
 .\"
-.\" 24st May 2022
+.\" July 6, 2022
 .\" Man page author:
 .\"   Tian Qiao <abcnsxyz@gmail.com>
 .\"
@@ -31,7 +31,7 @@ is maintained at:
 .I https://github.com/knownsec/pocsuite3/blob/master/docs/USAGE.md
 .PP
 .SH VERSION
-This manual page documents pocsuite3 version 1.9.5
+This manual page documents pocsuite3 version 1.9.6
 .SH AUTHOR
 .br
 (c) 2014-2022 by Knownsec 404 Team
diff --git a/manpages/pocsuite.1 b/manpages/pocsuite.1
@@ -1,6 +1,6 @@
-.TH POCSUITE "1" "May 2022" "Manual page for pocsuite"
+.TH POCSUITE "1" "July 2022" "Manual page for pocsuite"
 .\"
-.\" 24st May 2022
+.\" July 6, 2022
 .\" Man page author:
 .\"   Tian Qiao <abcnsxyz@gmail.com>
 .\"
@@ -90,7 +90,7 @@ HTTP Referer header value
 HTTP User\-Agent header value (default random)
 .TP
 \fB\-\-proxy\fR PROXY
-Use a proxy to connect to the target URL
+Use a proxy to connect to the target URL (protocol://host:port)
 .TP
 \fB\-\-proxy\-cred\fR PROXY_CRED
 Proxy authentication credentials (name:password)
@@ -200,6 +200,9 @@ Whether dork is in base64 format
 .IP
 Optimization options
 .TP
+\fB\-o\fR OUTPUT_PATH, \fB\-\-output\fR OUTPUT_PATH
+Output file to write (JSON Lines format)
+.TP
 \fB\-\-plugins\fR PLUGINS
 Load plugins to execute
 .TP
@@ -280,7 +283,7 @@ is maintained at:
 .I https://github.com/knownsec/pocsuite3/blob/master/docs/USAGE.md
 .PP
 .SH VERSION
-This manual page documents pocsuite3 version 1.9.5
+This manual page documents pocsuite3 version 1.9.6
 .SH AUTHOR
 .br
 (c) 2014-2022 by Knownsec 404 Team
diff --git a/pocsuite.ini b/pocsuite.ini
@@ -25,7 +25,7 @@ host =
 referer = 
 ; http user-agent header value (default random)
 agent = 
-; use a proxy to connect to the target url
+; use a proxy to connect to the target url (protocol://host:port)
 proxy = 
 ; proxy authentication credentials (name:password)
 proxy_cred = 
@@ -99,6 +99,8 @@ comparison = False
 dork_b64 = False
 
 [Optimization]
+; output file to write (json lines format)
+output_path =
 ; load plugins to execute
 plugins =
 ; user defined poc scripts path
diff --git a/pocsuite3/__init__.py b/pocsuite3/__init__.py
@@ -1,5 +1,5 @@
 __title__ = 'pocsuite3'
-__version__ = '1.9.5'
+__version__ = '1.9.6'
 __author__ = 'Knownsec 404 Team'
 __author_email__ = '404-team@knownsec.com'
 __license__ = 'GPLv2'
diff --git a/pocsuite3/lib/controller/controller.py b/pocsuite3/lib/controller/controller.py
@@ -17,7 +17,7 @@
 
 def runtime_check():
     if not kb.registered_pocs:
-        msg = "try 'pocsuite -h' or 'pocsuite --help' for more information"
+        msg = "No poc specified, try 'pocsuite -h' or 'pocsuite --help' for more information"
         logger.warn(msg)
         raise PocsuiteSystemException(msg)
 
diff --git a/pocsuite3/lib/core/common.py b/pocsuite3/lib/core/common.py
@@ -252,19 +252,14 @@ def parse_target_url(url):
     """
     Parse target URL
     """
-    ret = url
-
-    if conf.ipv6 and is_ipv6_address_format(url):
-        ret = "[" + ret + "]"
-
-    if not re.search("^http[s]*://", ret, re.I) and not re.search("^ws[s]*://", ret, re.I) and '://' not in ret:
-        port = urlparse(ret).port
-        if port and str(port).endswith('443'):
-            ret = "https://" + ret
-        else:
-            ret = "http://" + ret
+    try:
+        pr = urlparse(url)
+        if pr.scheme.lower() not in ['http', 'https', 'ws', 'wss']:
+            url = pr._replace(scheme='https' if str(pr.port).endswith('443') else 'http').geturl()
+    except ValueError:
+        pass
 
-    return ret
+    return url
 
 
 def is_url_format(value):
@@ -419,13 +414,18 @@ def parse_target(address, additional_ports=[]):
         pass
 
     targets.add(address)
-    pr = urlparse(address)
-    for port in additional_ports:
-        netloc = f'[{pr.hostname}]:{port}' if conf.ipv6 else f'{pr.hostname}:{port}'
-        t = pr._replace(netloc=netloc).geturl()
-        if t.startswith('tcp://'):
-            t = t.lstrip('tcp://')
-        targets.add(t)
+
+    try:
+        pr = urlparse(address)
+        for port in additional_ports:
+            netloc = f'[{pr.hostname}]:{port}' if conf.ipv6 else f'{pr.hostname}:{port}'
+            t = pr._replace(netloc=netloc).geturl()
+            if t.startswith('tcp://'):
+                t = t.lstrip('tcp://')
+            targets.add(t)
+    except ValueError:
+        pass
+
     return targets
 
 
diff --git a/pocsuite3/lib/core/option.py b/pocsuite3/lib/core/option.py
@@ -452,6 +452,9 @@ def _cleanup_options():
         conf.plugins = [i.strip() for i in conf.plugins]
         conf.plugins = list(set(conf.plugins))
 
+    if conf.output_path and 'file_record' not in conf.plugins:
+        conf.plugins.append('file_record')
+
     if conf.connect_back_port:
         conf.connect_back_port = int(conf.connect_back_port)
 
diff --git a/pocsuite3/lib/core/optiondict.py b/pocsuite3/lib/core/optiondict.py
@@ -59,6 +59,7 @@
         'dork_b64': 'boolean'
     },
     'Optimization': {
+        'output_path': 'string',
         'plugins': 'string',
         'pocs_path': 'string',
         'threads': 'integer',
diff --git a/pocsuite3/lib/core/poc.py b/pocsuite3/lib/core/poc.py
@@ -40,7 +40,8 @@ def __init__(self):
                                                       require=True)
             self.global_options["referer"] = OptString("", "HTTP Referer header value")
             self.global_options["agent"] = OptString("", "HTTP User-Agent header value")
-            self.global_options["proxy"] = OptString("", "Use a proxy to connect to the target URL")
+            self.global_options["proxy"] = OptString(
+                "", "Use a proxy to connect to the target URL (protocol://host:port)")
             self.global_options["timeout"] = OptInteger(10, "Seconds to wait before timeout connection (default 10)")
         else:
             self.global_options["rhost"] = OptString('', require=True)
@@ -143,10 +144,13 @@ def check_requirement(self, *args):
 
     def build_url(self):
         target = parse_target_url(self.target)
-        pr = urlparse(target)
-        self.scheme = 'https' if pr.scheme.startswith('https') else 'http'
-        self.rhost = pr.hostname
-        self.rport = pr.port if pr.port else 443 if pr.scheme.startswith('https') else 80
+        try:
+            pr = urlparse(target)
+            self.scheme = 'https' if pr.scheme.startswith('https') else 'http'
+            self.rhost = pr.hostname
+            self.rport = pr.port if pr.port else 443 if pr.scheme.startswith('https') else 80
+        except ValueError:
+            pass
         if self.target and self.current_protocol != POC_CATEGORY.PROTOCOL.HTTP and not conf.console_mode:
             self.setg_option("rport", self.rport)
             self.setg_option("rhost", self.rhost)
@@ -263,7 +267,7 @@ def _check(self, dork='', allow_redirects=False, return_obj=False, is_http=True,
                     self.url = url
                     logger.warn(f'auto correct url to: {mosaic(self.url)}')
                 break
-            except requests.ConnectionError:
+            except requests.RequestException:
                 pass
 
         if not self.url.startswith(self.scheme):
diff --git a/pocsuite3/lib/core/settings.py b/pocsuite3/lib/core/settings.py
@@ -160,6 +160,7 @@
     "tls",
     "comparison",
     "dork-b64",
+    "output_path",
     "plugins",
     "pocs-path",
     "threads",
diff --git a/pocsuite3/lib/parse/cmd.py b/pocsuite3/lib/parse/cmd.py
@@ -60,7 +60,8 @@ def cmd_line_parser(argv=None):
         request.add_argument("--host", dest="host", help="HTTP Host header value")
         request.add_argument("--referer", dest="referer", help="HTTP Referer header value")
         request.add_argument("--user-agent", dest="agent", help="HTTP User-Agent header value (default random)")
-        request.add_argument("--proxy", dest="proxy", help="Use a proxy to connect to the target URL")
+        request.add_argument("--proxy", dest="proxy",
+                             help="Use a proxy to connect to the target URL (protocol://host:port)")
         request.add_argument("--proxy-cred", dest="proxy_cred", help="Proxy authentication credentials (name:password)")
         request.add_argument("--timeout", dest="timeout", type=float, default=10,
                              help="Seconds to wait before timeout connection (default 10)")
@@ -123,6 +124,7 @@ def cmd_line_parser(argv=None):
 
         # Optimization options
         optimization = parser.add_argument_group("Optimization", "Optimization options")
+        optimization.add_argument("-o", "--output", dest="output_path", help="Output file to write (JSON Lines format)")
         optimization.add_argument("--plugins", dest="plugins", action="store", default=None,
                                   help="Load plugins to execute")
         optimization.add_argument("--pocs-path", dest="pocs_path", action="store", default=None,
diff --git a/pocsuite3/lib/request/patch/hook_request.py b/pocsuite3/lib/request/patch/hook_request.py
@@ -1,6 +1,6 @@
 from pocsuite3.lib.core.data import conf
 from pocsuite3.lib.core.enums import HTTP_HEADER
-from pocsuite3.lib.utils import generate_random_user_agent
+from pocsuite3.lib.utils import generate_random_user_agent, urlparse
 from requests.models import Request
 from requests.sessions import Session
 from requests.sessions import merge_cookies
@@ -40,6 +40,11 @@ def _merge_retain_none(request_setting, session_setting, dict_class=OrderedDict)
     if not conf.agent and HTTP_HEADER.USER_AGENT not in conf.http_headers:
         conf.http_headers[HTTP_HEADER.USER_AGENT] = generate_random_user_agent()
 
+    # Fix no connection adapters were found
+    pr = urlparse(url)
+    if pr.scheme.lower() not in ['http', 'https']:
+        url = pr._replace(scheme='https' if str(pr.port).endswith('443') else 'http').geturl()
+
     req = Request(
         method=method.upper(),
         url=url,
diff --git a/pocsuite3/modules/censys/__init__.py b/pocsuite3/modules/censys/__init__.py
@@ -78,7 +78,8 @@ def search(self, dork, pages=1, resource='ipv4'):
                 }
                 if cursor:
                     data['cursor'] = cursor
-                resp = requests.get(url, params=data, auth=(self.uid, self.secret), headers=self.headers)
+                resp = requests.get(url, params=data, auth=(self.uid, self.secret),
+                                    headers=self.headers, timeout=60)
                 if resp and resp.status_code == 200 and 'result' in resp.json():
                     results = resp.json()['result']['hits']
                     cursor = resp.json()['result']['links']['next']
diff --git a/pocsuite3/modules/fofa/__init__.py b/pocsuite3/modules/fofa/__init__.py
@@ -79,7 +79,7 @@ def search(self, dork, pages=1, resource='host'):
                     f"{self.api_url}/search/all?email={self.user}&key={self.token}&qbase64={dork}&"
                     f"fields={resource}&page={page}"
                 )
-                resp = requests.get(url, timeout=80, headers=self.headers)
+                resp = requests.get(url, headers=self.headers, timeout=60)
                 if resp and resp.status_code == 200 and "results" in resp.json():
                     content = resp.json()
                     for match in content['results']:
diff --git a/pocsuite3/modules/quake/__init__.py b/pocsuite3/modules/quake/__init__.py
@@ -69,7 +69,7 @@ def search(self, dork, pages=2):
                 time.sleep(1)
                 data['start'] = page
                 url = "https://quake.360.cn/api/v3/search/quake_service"
-                resp = requests.post(url, json=data, headers=self.headers, timeout=80)
+                resp = requests.post(url, json=data, headers=self.headers, timeout=60)
                 if resp and resp.status_code == 200 and resp.json()['code'] == 0:
                     content = resp.json()
                     for match in content['data']:
diff --git a/pocsuite3/modules/shodan/__init__.py b/pocsuite3/modules/shodan/__init__.py
@@ -67,7 +67,7 @@ def search(self, dork, pages=1):
             for page in range(1, pages + 1):
                 time.sleep(1)
                 url = f"{self.url}/shodan/{resource}/search?key={self.token}&query={dork}&page={page}"
-                resp = requests.get(url, headers=self.headers)
+                resp = requests.get(url, headers=self.headers, timeout=60)
                 if resp and resp.status_code == 200 and "total" in resp.json():
                     content = resp.json()
                     for match in content['matches']:
diff --git a/pocsuite3/modules/zoomeye/__init__.py b/pocsuite3/modules/zoomeye/__init__.py
@@ -72,7 +72,7 @@ def search(self, dork, pages=2, resource='host'):
             for page in range(1, pages + 1):
                 time.sleep(1)
                 url = f'{self.url}/{resource}/search?query={dork}&page={page}'
-                resp = requests.get(url, headers=self.headers)
+                resp = requests.get(url, headers=self.headers, timeout=60)
                 if resp and resp.status_code == 200 and 'matches' in resp.text:
                     content = resp.json()
 
diff --git a/pocsuite3/plugins/file_record.py b/pocsuite3/plugins/file_record.py
diff --git a/setup.py b/setup.py
diff --git a/tests/test_configfile.py b/tests/test_configfile.py
