diff --git a/.gitignore b/.gitignore index 57075a8..7d89efd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,134 @@ + +# Created by https://www.gitignore.io/api/python +# Edit at https://www.gitignore.io/?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# End of https://www.gitignore.io/api/python*.py[cod] .idea .vagrant Vagrantfile diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e5273f6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/usr/local/opt/python/bin/python3.7" +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b8b9909 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# ngxtop + +[![github-release](https://badgen.net/badge/github/release/cyan?icon=github)](https://github.com/dalmatele/ngxtop/releases) \ No newline at end of file diff --git a/README.rst b/README.rst index b84550e..cfceadd 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,21 @@ Installation :: - pip install ngxtop + Requirement: + python + pip + setuptools + wheel + tqdm + twine + How to compile: + In the root folder of project, you run this command: + python setup.py bdist_wheel + How to install: + In the root folder of project, you run this command: + python -m pip install dist/ngxtop-0.1.2-py2.py3-none-any.whl + Please change ngxtop-0.1.2-py2.py3-none-any.whl by your file. + How to run: please run ngxtop --help to know more information Note: ``ngxtop`` is primarily developed and tested with python2 but also supports python3. diff --git a/ngxtop.egg-info/PKG-INFO b/ngxtop.egg-info/PKG-INFO new file mode 100644 index 0000000..7bfcc18 --- /dev/null +++ b/ngxtop.egg-info/PKG-INFO @@ -0,0 +1,175 @@ +Metadata-Version: 1.1 +Name: ngxtop +Version: 0.0.3 +Summary: Real-time metrics for nginx server +Home-page: https://github.com/lebinh/ngxtop +Author: Binh Le +Author-email: lebinh.it@gmail.com +License: MIT +Description: ================================================================ + ``ngxtop`` - **real-time** metrics for nginx server (and others) + ================================================================ + + **ngxtop** parses your nginx access log and outputs useful, ``top``-like, metrics of your nginx server. + So you can tell what is happening with your server in real-time. + + ``ngxtop`` is designed to run in a short-period time just like the ``top`` command for troubleshooting and monitoring + your Nginx server at the moment. If you need a long running monitoring process or storing your webserver stats in external + monitoring / graphing system, you can try `Luameter `_. + + ``ngxtop`` tries to determine the correct location and format of nginx access log file by default, so you can just run + ``ngxtop`` and having a close look at all requests coming to your nginx server. But it does not limit you to nginx + and the default top view. ``ngxtop`` is flexible enough for you to configure and change most of its behaviours. + You can query for different things, specify your log and format, even parse remote Apache common access log with ease. + See sample usages below for some ideas about what you can do with it. + + Installation + ------------ + + :: + + pip install ngxtop + + + Note: ``ngxtop`` is primarily developed and tested with python2 but also supports python3. + + Usage + ----- + + :: + + Usage: + ngxtop [options] + ngxtop [options] (print|top|avg|sum) + ngxtop info + + Options: + -l , --access-log access log file to parse. + -f , --log-format log format as specify in log_format directive. + --no-follow ngxtop default behavior is to ignore current lines in log + and only watch for new lines as they are written to the access log. + Use this flag to tell ngxtop to process the current content of the access log instead. + -t , --interval report interval when running in follow mode [default: 2.0] + + -g , --group-by group by variable [default: request_path] + -w , --having having clause [default: 1] + -o , --order-by order of output for default query [default: count] + -n , --limit limit the number of records included in report for top command [default: 10] + -a ..., --a ... add exp (must be aggregation exp: sum, avg, min, max, etc.) into output + + -v, --verbose more verbose output + -d, --debug print every line and parsed record + -h, --help print this help message. + --version print version information. + + Advanced / experimental options: + -c , --config allow ngxtop to parse nginx config file for log format and location. + -i , --filter filter in, records satisfied given expression are processed. + -p , --pre-filter in-filter expression to check in pre-parsing phase. + + Samples + ------- + + Default output + ~~~~~~~~~~~~~~ + + :: + + $ ngxtop + running for 411 seconds, 64332 records processed: 156.60 req/sec + + Summary: + | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |---------+------------------+-------+-------+-------+-------| + | 64332 | 2775.251 | 61262 | 2994 | 71 | 5 | + + Detailed: + | request_path | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |------------------------------------------+---------+------------------+-------+-------+-------+-------| + | /abc/xyz/xxxx | 20946 | 434.693 | 20935 | 0 | 11 | 0 | + | /xxxxx.json | 5633 | 1483.723 | 5633 | 0 | 0 | 0 | + | /xxxxx/xxx/xxxxxxxxxxxxx | 3629 | 6835.499 | 3626 | 0 | 3 | 0 | + | /xxxxx/xxx/xxxxxxxx | 3627 | 15971.885 | 3623 | 0 | 4 | 0 | + | /xxxxx/xxx/xxxxxxx | 3624 | 7830.236 | 3621 | 0 | 3 | 0 | + | /static/js/minified/utils.min.js | 3031 | 1781.155 | 2104 | 927 | 0 | 0 | + | /static/js/minified/xxxxxxx.min.v1.js | 2889 | 2210.235 | 2068 | 821 | 0 | 0 | + | /static/tracking/js/xxxxxxxx.js | 2594 | 1325.681 | 1927 | 667 | 0 | 0 | + | /xxxxx/xxx.html | 2521 | 573.597 | 2520 | 0 | 1 | 0 | + | /xxxxx/xxxx.json | 1840 | 800.542 | 1839 | 0 | 1 | 0 | + + View top source IPs of clients + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + $ ngxtop top remote_addr + running for 20 seconds, 3215 records processed: 159.62 req/sec + + top remote_addr + | remote_addr | count | + |-----------------+---------| + | 118.173.177.161 | 20 | + | 110.78.145.3 | 16 | + | 171.7.153.7 | 16 | + | 180.183.67.155 | 16 | + | 183.89.65.9 | 16 | + | 202.28.182.5 | 16 | + | 1.47.170.12 | 15 | + | 119.46.184.2 | 15 | + | 125.26.135.219 | 15 | + | 125.26.213.203 | 15 | + + List 4xx or 5xx responses together with HTTP referer + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + $ ngxtop -i 'status >= 400' print request status http_referer + running for 2 seconds, 28 records processed: 13.95 req/sec + + request, status, http_referer: + | request | status | http_referer | + |-----------+----------+----------------| + | - | 400 | - | + + Parse apache log from remote server with `common` format + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + $ ssh user@remote_server tail -f /var/log/apache2/access.log | ngxtop -f common + running for 20 seconds, 1068 records processed: 53.01 req/sec + + Summary: + | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |---------+------------------+-------+-------+-------+-------| + | 1068 | 28026.763 | 1029 | 20 | 19 | 0 | + + Detailed: + | request_path | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |------------------------------------------+---------+------------------+-------+-------+-------+-------| + | /xxxxxxxxxx | 199 | 55150.402 | 199 | 0 | 0 | 0 | + | /xxxxxxxx/xxxxx | 167 | 47591.826 | 167 | 0 | 0 | 0 | + | /xxxxxxxxxxxxx/xxxxxx | 25 | 7432.200 | 25 | 0 | 0 | 0 | + | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxxx | 22 | 698.727 | 22 | 0 | 0 | 0 | + | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxx | 19 | 7431.632 | 19 | 0 | 0 | 0 | + | /xxxxx/xxxxx/ | 18 | 7840.889 | 18 | 0 | 0 | 0 | + | /xxxxxxxx/xxxxxxxxxxxxxxxxx | 15 | 7356.000 | 15 | 0 | 0 | 0 | + | /xxxxxxxxxxx/xxxxxxxx | 15 | 9978.800 | 15 | 0 | 0 | 0 | + | /xxxxx/ | 14 | 0.000 | 0 | 14 | 0 | 0 | + | /xxxxxxxxxx/xxxxxxxx/xxxxx | 13 | 20530.154 | 13 | 0 | 0 | 0 | + + +Keywords: cli monitoring nginx system +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: License :: OSI Approved :: MIT License +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.2 +Classifier: Programming Language :: Python :: 3.3 diff --git a/ngxtop.egg-info/SOURCES.txt b/ngxtop.egg-info/SOURCES.txt new file mode 100644 index 0000000..0eb294a --- /dev/null +++ b/ngxtop.egg-info/SOURCES.txt @@ -0,0 +1,13 @@ +README.rst +setup.cfg +setup.py +ngxtop/__init__.py +ngxtop/config_parser.py +ngxtop/ngxtop.py +ngxtop/utils.py +ngxtop.egg-info/PKG-INFO +ngxtop.egg-info/SOURCES.txt +ngxtop.egg-info/dependency_links.txt +ngxtop.egg-info/entry_points.txt +ngxtop.egg-info/requires.txt +ngxtop.egg-info/top_level.txt \ No newline at end of file diff --git a/ngxtop.egg-info/dependency_links.txt b/ngxtop.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ngxtop.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/ngxtop.egg-info/entry_points.txt b/ngxtop.egg-info/entry_points.txt new file mode 100644 index 0000000..f8a510e --- /dev/null +++ b/ngxtop.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +ngxtop = ngxtop.ngxtop:main + diff --git a/ngxtop.egg-info/requires.txt b/ngxtop.egg-info/requires.txt new file mode 100644 index 0000000..5691c21 --- /dev/null +++ b/ngxtop.egg-info/requires.txt @@ -0,0 +1,3 @@ +docopt +tabulate +pyparsing diff --git a/ngxtop.egg-info/top_level.txt b/ngxtop.egg-info/top_level.txt new file mode 100644 index 0000000..103e23a --- /dev/null +++ b/ngxtop.egg-info/top_level.txt @@ -0,0 +1 @@ +ngxtop diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py index 0d9f00a..0445acc 100644 --- a/ngxtop/config_parser.py +++ b/ngxtop/config_parser.py @@ -13,9 +13,12 @@ REGEX_SPECIAL_CHARS = r'([\.\*\+\?\|\(\)\{\}\[\]])' REGEX_LOG_FORMAT_VARIABLE = r'\$([a-zA-Z0-9\_]+)' +# LOG_FORMAT_COMBINED = '$remote_addr - $remote_user [$time_local] ' \ +# '"$request" $status $body_bytes_sent ' \ +# '"$http_referer" "$http_user_agent"' LOG_FORMAT_COMBINED = '$remote_addr - $remote_user [$time_local] ' \ - '"$request" $status $body_bytes_sent ' \ - '"$http_referer" "$http_user_agent"' + '"$request" $status $body_bytes_sent ' \ + '"$http_referer" "$http_user_agent" $rt $uct $uht $urt $cache' LOG_FORMAT_COMMON = '$remote_addr - $remote_user [$time_local] ' \ '"$request" $status $body_bytes_sent ' \ '"$http_x_forwarded_for"' diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 8667b8b..a402650 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -73,7 +73,6 @@ from docopt import docopt import tabulate - from .config_parser import detect_log_config, detect_config_path, extract_variables, build_pattern from .utils import error_exit @@ -86,7 +85,9 @@ count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', - count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' + count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx', + count(CASE WHEN cache_status = 1 THEN 1 END) as 'cached', + count(CASE WHEN cache_status = 0 THEN 1 END) as 'non cached' FROM log ORDER BY %(--order-by)s DESC LIMIT %(--limit)s'''), @@ -99,7 +100,9 @@ count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', - count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' + count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx', + count(CASE WHEN cache_status = 1 THEN 1 END) as 'cached', + count(CASE WHEN cache_status = 0 THEN 1 END) as 'non cached' FROM log GROUP BY %(--group-by)s HAVING %(--having)s @@ -107,7 +110,7 @@ LIMIT %(--limit)s''') ] -DEFAULT_FIELDS = set(['status_type', 'bytes_sent']) +DEFAULT_FIELDS = set(['status_type', 'bytes_sent', 'cache_status']) # ====================== @@ -181,6 +184,12 @@ def to_int(value): def to_float(value): return float(value) if value and value != '-' else 0.0 +def hit_or_miss(record): + if(record["cache"].find("HIT") != -1): + return 1 + else: + return 0 + def parse_log(lines, pattern): matches = (pattern.match(l) for l in lines) @@ -191,6 +200,7 @@ def parse_log(lines, pattern): records = map_field('bytes_sent', to_int, records) records = map_field('request_time', to_float, records) records = add_field('request_path', parse_request_path, records) + records = add_field('cache_status', hit_or_miss, records) return records @@ -342,7 +352,7 @@ def print_report(sig, frame): signal.setitimer(signal.ITIMER_REAL, 0.1, interval) -def process(arguments): +def process(arguments): access_log = arguments['--access-log'] log_format = arguments['--log-format'] if access_log is None and not sys.stdin.isatty(): @@ -365,13 +375,15 @@ def process(arguments): source = build_source(access_log, arguments) pattern = build_pattern(log_format) + logging.debug("ducla") + logging.debug(pattern) processor = build_processor(arguments) setup_reporter(processor, arguments) process_log(source, pattern, processor, arguments) def main(): - args = docopt(__doc__, version='xstat 0.1') + args = docopt(__doc__, version='xstat 0.3.1') log_level = logging.WARNING if args['--verbose']: diff --git a/setup.py b/setup.py index 6823c7c..1bec1e2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='ngxtop', - version='0.0.3', + version='0.0.4', description='Real-time metrics for nginx server', long_description=open('README.rst').read(), license='MIT', diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 3b6504b..383ced5 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -1,3 +1,6 @@ +import re +import json +import logging from ngxtop import config_parser @@ -17,12 +20,13 @@ def test_get_log_formats(): assert 'main' in formats assert "'$http_referer'" in formats['main'] assert 'te st' in formats + print("Ok") def test_get_access_logs_no_format(): config = ''' http { - # ubuntu default + # ubngxuntu default access_log /var/log/nginx/access.log; # syslog is a valid access log, but we can't follow it @@ -58,3 +62,63 @@ def test_access_logs_with_format_name(): assert len(logs) == 2 assert logs['/path/to/main.log'] == 'main' assert logs['/path/to/test.log'] == 'te st' + +def hit_or_miss(record): + if(record["cache_status"].sub("HIT") != 0): + return 1 + else: + return 0 + +def map_field(field, func, dict_sequence): + """ + Apply given function to value of given key in every dictionary in sequence and + set the result as new value for that key. + """ + for item in dict_sequence: + try: + item[field] = func(item.get(field, None)) + yield item + except ValueError: + pass + +def add_field(field, func, dict_sequence): + """ + Apply given function to the record and store result in given field of current record. + Do nothing if record already contains given field. + """ + for item in dict_sequence: + if field not in item: + item[field] = func(item) + yield item + +def hit_or_miss(record): + logging.info(record["cache"]) + print(record["cache"]) + if(record["cache"].find("HIT") != 0): + return 1 + else: + return 0 + +if __name__ == "__main__": + REGEX_SPECIAL_CHARS = r'([\.\*\+\?\|\(\)\{\}\[\]])' + REGEX_LOG_FORMAT_VARIABLE = r'\$([a-zA-Z0-9\_]+)' + LOG_FORMAT_COMBINED = '$remote_addr - $remote_user [$time_local] ' \ + '"$request" $status $body_bytes_sent ' \ + '"$http_referer" "$http_user_agent" $rt $uct $uht $urt $cache' + LOG_FORMAT_COMMON = '$remote_addr - $remote_user [$time_local] ' \ + '"$request" $status $body_bytes_sent ' \ + '"$http_x_forwarded_for"' + log_format = LOG_FORMAT_COMBINED + pattern = config_parser.build_pattern(log_format) + # pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format) + # print pattern + # pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) + # print pattern + source = '100.101.11.197 - - [26/Jul/2019:04:54:00 +0000] "GET /a9383d04d7d0420bae10dbf96bb27d9b-stream/d43cf4f8-11af-4947-842a-a488050081f1/package/audio/unk-1/mp4a/51411/51411-2-15.m4s HTTP/2.0" 200 12694 "https://sdk.uiza.io/v3/index.html" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/80.0.180 Chrome/74.0.3729.180 Safari/537.36" rt=0.000 uct="-" uht="-" urt="-" uc="HIT" 116.104.33.174 - - [26/Jul/2019:04:54:00 +0000] "GET /9521cff34e86473095644ba71cbd0e7f-live/48a629d7-4198-4369-b62a-fdb835f4f129/9521cff34e86473095644ba71cbd0e7f-live/b-v1400-a128/dvr_v_p1_1669582.ts HTTP/1.1" 200 193076 "-" "AppleCoreMedia/1.0.0.14G60 (iPhone; U; CPU OS 10_3_3 like Mac OS X; pt_br)" rt=0.136 uct="0.060, 0.000" uht="0.120, 0.012" urt="0.120, 0.016" ' + line = pattern.match(source) + record = line.groupdict() + hit_or_miss(record) + # record = add_field('cache_status', hit_or_miss, record) + print record + +