Merge pull request #332 from knownsec/feat/yaml

feat: yaml poc support

Author: 13ph03nix

CHANGELOG.md

@@ -1,3 +1,9 @@
+# version 2.0.0
+----------------
+* yaml poc support, compatible with nuclei
+* fix httpserver module hangs on macos platform
+* auto correction of url protocol based on status code
+
 # version 1.9.11
 ----------------
 * support customize poc protocol and default port #321

manpages/poc-console.1

@@ -31,7 +31,7 @@ is maintained at:
 .I https://pocsuite.org
 .PP
 .SH VERSION
-This manual page documents pocsuite3 version 1.9.11
+This manual page documents pocsuite3 version 2.0.0
 .SH AUTHOR
 .br
 (c) 2014-present by Knownsec 404 Team

manpages/pocsuite.1

@@ -286,7 +286,7 @@ is maintained at:
 .I https://pocsuite.org
 .PP
 .SH VERSION
-This manual page documents pocsuite3 version 1.9.11
+This manual page documents pocsuite3 version 2.0.0
 .SH AUTHOR
 .br
 (c) 2014-present by Knownsec 404 Team

pocsuite3/__init__.py

@@ -1,5 +1,5 @@
 __title__ = 'pocsuite3'
-__version__ = '1.9.11'
+__version__ = '2.0.0'
 __author__ = 'Knownsec 404 Team'
 __author_email__ = '[email protected]'
 __license__ = 'GPLv2'

pocsuite3/api/__init__.py

@@ -1,10 +1,25 @@
+import base64
+import binascii
+import collections
+import json
+import os
+import re
+import socket
+import ssl
+import struct
+import textwrap
+import time
+import urllib
+import zlib
+
 from pocsuite3.lib.controller.controller import start
-from pocsuite3.lib.core.common import (encoder_bash_payload, check_port,
+from pocsuite3.lib.core.common import (OrderedDict, OrderedSet, check_port,
+                                       encoder_bash_payload,
                                        encoder_powershell_payload, get_host_ip,
-                                       get_host_ipv6, single_time_warn_message)
+                                       get_host_ipv6, mosaic,
+                                       single_time_warn_message, urlparse)
 from pocsuite3.lib.core.data import conf, kb, logger, paths
 from pocsuite3.lib.core.datatype import AttribDict
-from pocsuite3.lib.core.common import OrderedSet, OrderedDict, mosaic, urlparse
 from pocsuite3.lib.core.enums import PLUGIN_TYPE, POC_CATEGORY, VUL_TYPE
 from pocsuite3.lib.core.interpreter_option import (OptBool, OptDict, OptFloat,
                                                    OptInteger, OptIP, OptItems,
@@ -17,37 +32,24 @@
 from pocsuite3.lib.core.settings import DEFAULT_LISTENER_PORT
 from pocsuite3.lib.request import requests
 from pocsuite3.lib.utils import (generate_shellcode_list, get_middle_text,
-                                 random_str, minimum_version_required)
+                                 minimum_version_required, random_str)
+from pocsuite3.lib.yaml.nuclei import Nuclei
 from pocsuite3.modules.censys import Censys
 from pocsuite3.modules.ceye import CEye
 from pocsuite3.modules.fofa import Fofa
 from pocsuite3.modules.httpserver import PHTTPServer
-from pocsuite3.modules.listener import (REVERSE_PAYLOAD, BIND_PAYLOAD, bind_shell,
-                                        bind_tcp_shell, bind_telnet_shell)
-from pocsuite3.modules.quake import Quake
 from pocsuite3.modules.hunter import Hunter
+from pocsuite3.modules.interactsh import Interactsh
+from pocsuite3.modules.listener import (BIND_PAYLOAD, REVERSE_PAYLOAD,
+                                        bind_shell, bind_tcp_shell,
+                                        bind_telnet_shell)
+from pocsuite3.modules.quake import Quake
 from pocsuite3.modules.seebug import Seebug
 from pocsuite3.modules.shodan import Shodan
 from pocsuite3.modules.spider import crawl
 from pocsuite3.modules.zoomeye import ZoomEye
-from pocsuite3.modules.interactsh import Interactsh
 from pocsuite3.shellcodes import OSShellcodes, WebShell
 
-__all__ = ('requests', 'PluginBase', 'register_plugin', 'PLUGIN_TYPE',
-           'POCBase', 'Output', 'AttribDict', 'POC_CATEGORY', 'VUL_TYPE',
-           'register_poc', 'conf', 'kb', 'logger', 'paths', 'minimum_version_required',
-           'DEFAULT_LISTENER_PORT', 'load_file_to_module', 'OrderedDict', 'OrderedSet',
-           'load_string_to_module', 'single_time_warn_message', 'CEye',
-           'Seebug', 'ZoomEye', 'Shodan', 'Fofa', 'Quake', 'Hunter', 'Censys',
-           'PHTTPServer', 'REVERSE_PAYLOAD', 'BIND_PAYLOAD', 'get_listener_ip', 'mosaic',
-           'urlparse', 'get_listener_port', 'get_results', 'init_pocsuite',
-           'start_pocsuite', 'get_poc_options', 'crawl', 'OSShellcodes',
-           'WebShell', 'OptDict', 'OptIP', 'OptPort', 'OptBool', 'OptInteger',
-           'OptFloat', 'OptString', 'OptItems', 'get_middle_text',
-           'generate_shellcode_list', 'random_str', 'encoder_bash_payload', 'check_port',
-           'encoder_powershell_payload', 'get_host_ip', 'get_host_ipv6', 'bind_shell',
-           'bind_tcp_shell', 'bind_telnet_shell', 'Interactsh')
-
 
 def get_listener_ip():
     return conf.connect_back_host

pocsuite3/lib/core/common.py

@@ -39,7 +39,6 @@
 from pocsuite3.lib.core.settings import IP_ADDRESS_REGEX
 from pocsuite3.lib.core.settings import OLD_VERSION_CHARACTER
 from pocsuite3.lib.core.settings import POCSUITE_VERSION_CHARACTER
-from pocsuite3.lib.core.settings import POC_NAME_REGEX
 from pocsuite3.lib.core.settings import POC_REQUIRES_REGEX
 from pocsuite3.lib.core.settings import UNICODE_ENCODING
 from pocsuite3.lib.core.settings import URL_ADDRESS_REGEX
@@ -576,7 +575,11 @@ def get_poc_requires(code):
 
 
 def get_poc_name(code):
-    return extract_regex_result(POC_NAME_REGEX, code)
+    if re.search(r'register_poc', code):
+        return extract_regex_result(r"""(?sm)POCBase\):.*?name\s*=\s*['"](?P<result>.*?)['"]""", code)
+    elif re.search(r'matchers:\s*-', code):
+        return extract_regex_result(r"""(?sm)\s*name\s*:\s*(?P<result>[^\n]*).*matchers:""", code)
+    return ''
 
 
 def is_os_64bit():
@@ -897,7 +900,7 @@ def index_modules(modules_directory):
 
     modules = []
     for root, _, files in os.walk(modules_directory):
-        files = filter(lambda x: not x.startswith("__") and x.endswith(".py"), files)
+        files = filter(lambda x: not x.startswith("__") and x.endswith(".py") or x.endswith(".yaml"), files)
         modules.extend(map(lambda x: os.path.join(root, os.path.splitext(x)[0]), files))
 
     return modules

pocsuite3/lib/core/interpreter.py

@@ -1,8 +1,4 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# @Time    : 2018/12/25 上午10:58
-# @Author  : chenghs
-# @File    : interpreter.py
+# pylint: disable=E0202
 import os
 import re
 import chardet
@@ -325,19 +321,27 @@ def command_use(self, module_path, *args, **kwargs):
                 logger.warning("Index out of range")
                 return
             module_path = self.last_search[index]
-        if not module_path.endswith(".py"):
-            module_path = module_path + ".py"
-        if not os.path.exists(module_path):
-            module_path = os.path.join(self.module_parent_directory, module_path)
-            if not os.path.exists(module_path):
-                errMsg = "No such file: '{0}'".format(module_path)
-                logger.error(errMsg)
-                return
+
+        module_ext = ''
+        module_path_found = False
+        for module_ext in ['.py', '.yaml']:
+            if os.path.exists(module_path + module_ext):
+                module_path_found = True
+                break
+            elif os.path.exists(os.path.join(self.module_parent_directory, module_path + module_ext)):
+                module_path_found = True
+                module_path = os.path.join(self.module_parent_directory, module_path + module_ext)
+                break
+
+        if not module_path_found:
+            errMsg = "No such file: '{0}'".format(module_path)
+            logger.error(errMsg)
+            return
+
         try:
             load_file_to_module(module_path)
             self.current_module = kb.current_poc
-            self.current_module.pocsuite3_module_path = ltrim(
-                rtrim(module_path, ".py"), self.module_parent_directory)
+            self.current_module.pocsuite3_module_path = ltrim(rtrim(module_path, module_ext), self.module_parent_directory)
         except Exception as err:
             logger.error(str(err))
 
@@ -457,6 +461,8 @@ def command_list(self, *args, **kwargs):
         index = 0
         for tmp_module in self.main_modules_dirs:
             found = os.path.join(self.module_parent_directory, tmp_module + ".py")
+            if not os.path.exists(found):
+                found = os.path.join(self.module_parent_directory, tmp_module + ".yaml")
             code = get_file_text(found)
             name = get_poc_name(code)
             tb.add_row([str(index), tmp_module, name])

pocsuite3/lib/core/option.py

@@ -338,20 +338,21 @@ def _set_pocs_modules():
 
                 elif any([poc in exists_poc_with_ext, poc in exists_pocs]):
                     poc_name, poc_ext = os.path.splitext(poc)
-                    if poc_ext in ['.py', '.pyc']:
+                    if poc_ext in ['.py', '.pyc', '.yaml']:
                         file_path = os.path.join(paths.POCSUITE_POCS_PATH, poc)
                     else:
                         file_path = os.path.join(paths.POCSUITE_POCS_PATH, poc + exists_pocs.get(poc))
                     _pocs.append(file_path)
 
                 elif check_path(poc):
                     for root, _, files in os.walk(poc):
-                        files = filter(lambda x: not x.startswith("__") and x.endswith(".py"), files)
+                        files = filter(lambda x: not x.startswith("__") and x.endswith(".py") or
+                                       x.endswith('.yaml'), files)
                         _pocs.extend(map(lambda x: os.path.join(root, x), files))
 
                 for p in _pocs:
                     file_content = get_file_text(p)
-                    if not re.search(r'register_poc', file_content):
+                    if not re.search(r'register_poc|matchers:\s+-', file_content):
                         continue
                     if conf.poc_keyword:
                         if not re.search(conf.poc_keyword, file_content, re.I | re.M):

pocsuite3/lib/core/register.py

@@ -68,6 +68,12 @@ def check_requires(data):
     def exec_module(self, module):
         filename = self.get_filename(self.fullname)
         poc_code = self.get_data(filename)
+
+        # convert yaml template to pocsuite3 poc script
+        if filename.endswith('.yaml') and re.search(r'matchers:\s+-', poc_code):
+            from pocsuite3.lib.yaml.nuclei import Nuclei
+            poc_code = str(Nuclei(poc_code))
+
         self.check_requires(poc_code)
         obj = compile(poc_code, filename, 'exec', dont_inherit=True, optimize=-1)
         try:

pocsuite3/lib/core/settings.py

@@ -93,8 +93,6 @@
 
 POC_REQUIRES_REGEX = r"install_requires\s*=\s*\[(?P<result>.*?)\]"
 
-POC_NAME_REGEX = r"""(?sm)POCBase\):.*?name\s*=\s*['"](?P<result>.*?)['"]"""
-
 MAX_NUMBER_OF_THREADS = 200
 
 DEFAULT_LISTENER_PORT = 6666