diff --git a/.github/.yamllint.yml b/.github/.yamllint.yml new file mode 100644 index 0000000..e66ccad --- /dev/null +++ b/.github/.yamllint.yml @@ -0,0 +1,2 @@ +rules: + line-length: disable \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index b018957..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,30 +0,0 @@ -on: push -name: Build ChopChop -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.14.x - - name: Checkout code - uses: actions/checkout@v2 - - name: Unit Tests - run: go test ./... - - name: Install gox - run: go get github.com/mitchellh/gox - - name: Build using gox - run: gox -ldflags "-X main.Version=$BUILD_VERSION -X main.BuildDate=$BUILD_DATE" -output "dist/ChopChop_{{.OS}}_{{.Arch}}" - - name: Upload ChopChop builds - uses: actions/upload-artifact@v2 - with: - name: chopchop-artifacts - path: dist/* - - name: Release - uses: fnkr/github-action-ghr@v1 - if: startsWith(github.ref, 'refs/tags/') - env: - GHR_COMPRESS: gz - GHR_PATH: dist/ - GITHUB_TOKEN: ${{ secrets.DEPLOY_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ce5beb3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,161 @@ +name: CI + +on: [push, pull_request] + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - name: Cancel previous + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + + unit-tests: + strategy: + matrix: + go-version: [1.x, 1.16.x] + platform: [ubuntu-latest, macos-latest, windows-latest] + include: + - go-version: 1.x + platform: ubuntu-latest + update-coverage: true + runs-on: ${{ matrix.platform }} + needs: [setup] + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - name: Cache go modules + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-go- + + - name: Run go fmt + if: runner.os != 'Windows' + run: diff -u <(echo -n) <(gofmt -d -s .) + + - name: Ensure go generate produces a zero diff + shell: bash + run: go generate -x ./... && git diff --exit-code; code=$?; git checkout -- .; (exit $code) + + - name: Run go vet + run: go vet ./... + + - name: Run go test + run: go test -v -race -coverprofile coverage.txt ./... + + - name: Upload coverage to Codecov + if: ${{ matrix.update-coverage }} + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + chopchop-endpoint: + runs-on: ubuntu-latest + needs: [setup] + steps: + - uses: actions/checkout@v1 + - run: | + cat chopchop.yml | grep "uri:" | sort | uniq -c | sort -n + test=`cat chopchop.yml | grep "endpoint:" | sort | uniq -c | grep -v 1 | wc -l` + if [ $test != 0 ]; then echo "There shouldn't be multiple (and identical) 'endpoint'. It should be refactored. "; exit 1; fi + + go-lint: + runs-on: ubuntu-latest + needs: [setup] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: 1.16.x + + - name: go-lint + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.39.0 + golangci-lint run + + yaml-lint: + runs-on: ubuntu-latest + needs: [setup] + steps: + - uses: actions/checkout@v2 + - name: yaml-lint + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: chopchop.yml + config_file: .github/.yamllint.yml + + functional-tests: + runs-on: ubuntu-latest + needs: [unit-tests, chopchop-endpoint, go-lint, yaml-lint] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: 1.16.x + + - name: Install RobotFramework + run: pip install robotframework + + - name: Run RobotFramework tests + run: | + cd robot + ./run.sh + + - name: Upload Robot outputs + uses: actions/upload-artifact@v2 + with: + name: robot-output + path: robot/out/* + + build-and-publish: + runs-on: ubuntu-latest + needs: [functional-tests] + if: ${{ github.event_name == 'push' }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.x + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Cache go modules + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-go- + + - name: Install gox + run: go get github.com/mitchellh/gox + + - name: Build using gox + run: | + cd cmd + gox -ldflags "-X main.Version=$BUILD_VERSION -X main.BuildDate=$BUILD_DATE" \ + -output "../artifacts/ChopChop_{{.OS}}_{{.Arch}}" \ + -osarch="!darwin/386" + + - name: Upload ChopChop builds + uses: actions/upload-artifact@v2 + with: + name: chopchop-artifacts + path: artifacts/* + + - name: Release + uses: fnkr/github-action-ghr@v1 + if: startsWith(github.ref, 'refs/tags/') + env: + GHR_COMPRESS: gz + GHR_PATH: artifacts/ + GITHUB_TOKEN: ${{ secrets.DEPLOY_TOKEN }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index fe50703..d663b4b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,53 +10,23 @@ on: tags: - v* - # Run tests for any PRs. - pull_request: - env: IMAGE_NAME: gochopchop jobs: - # Run tests. - # See also https://docs.docker.com/docker-hub/builds/automated-testing/ - test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Run tests - run: | - if [ -f docker-compose.test.yml ]; then - docker-compose --file docker-compose.test.yml build - docker-compose --file docker-compose.test.yml run sut - else - docker build . --file Dockerfile - fi - # Push image to GitHub Packages. # See also https://docs.docker.com/docker-hub/builds/ push: - # Ensure test job passes before pushing image. - needs: test - runs-on: ubuntu-latest - if: github.event_name == 'push' - steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.14.x + - name: Log into GitHub Container Registry + # The CR_PAT secret is a PAT with `read:packages` and `write:packages` scopes + run: echo "${{ secrets.CR_PAT }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - uses: actions/checkout@v2 - - name: Unit Tests - run: go test ./... - - name: Build image - run: docker build . --file Dockerfile --tag $IMAGE_NAME - - name: Log into GitHub Container Registry - # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` - run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + - name: Build image + run: docker build -t $IMAGE_NAME . - name: Push image to GitHub Container Registry run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 59f77fd..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: ChopChop YAML configuration Linter - -on: [push] - -jobs: - lintAllTheThings: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: yaml-lint - uses: ibiqlik/action-yamllint@v1 - with: - file_or_dir: chopchop.yml - config_file: .yamllint.yml - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: | - cat chopchop.yml | grep "uri:" | sort | uniq -c | sort -n - test=`cat chopchop.yml | grep "endpoint:" | sort | uniq -c | grep -v 1 | wc -l` - if [ $test != 0 ]; then echo "There shouldn't be multiple (and identical) 'endpoint'. It should be refactored. "; exit 1; fi diff --git a/.gitignore b/.gitignore index da21c80..67d71c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,132 +1,7 @@ +# IDE +/.idea/** +/.vscode/** -# Created by https://www.toptal.com/developers/gitignore/api/go,jetbrains -# Edit at https://www.toptal.com/developers/gitignore?templates=go,jetbrains - -### Go ### -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test -*.txt -*.json -*.csv - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -### Go Patch ### -/vendor/ -/Godeps/ - -### JetBrains ### -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 - -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - -### JetBrains Patch ### -# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 - -# *.iml -# modules.xml -# .idea/misc.xml -# *.ipr - -# Sonarlint plugin -# https://plugins.jetbrains.com/plugin/7973-sonarlint -.idea/**/sonarlint/ - -# SonarQube Plugin -# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin -.idea/**/sonarIssues.xml - -# Markdown Navigator plugin -# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced -.idea/**/markdown-navigator.xml -.idea/**/markdown-navigator-enh.xml -.idea/**/markdown-navigator/ - -# Cache file creation bug -# See https://youtrack.jetbrains.com/issue/JBR-2257 -.idea/$CACHE_FILE$ - -# CodeStream plugin -# https://plugins.jetbrains.com/plugin/12206-codestream -.idea/codestream.xml - -# End of https://www.toptal.com/developers/gitignore/api/go,jetbrains +# RobotFramework +**/__pycache__/** +/robot/out/** \ No newline at end of file diff --git a/.yamllint.yml b/.yamllint.yml deleted file mode 100644 index 52e823b..0000000 --- a/.yamllint.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - rules: - line-length: disable \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index bc3f9de..4702ec3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,16 @@ -FROM golang:1.13 AS build -RUN mkdir /app -ADD . /app/ -WORKDIR /app +# Build stage +FROM golang:1.16 AS builder +WORKDIR /go/src COPY go.mod go.sum ./ RUN go mod download -COPY chopchop.yml ./ -RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build . -CMD ["/app/gochopchop"] +COPY . . +ENV GOOS=linux +ENV GOARCH=amd64 +ENV CGO_ENABLED=0 +RUN go build -o /go/bin/gochopchop cmd/main.go +# Prod stage FROM alpine:3.8 -RUN mkdir -p /tmp -COPY --from=build /app/gochopchop /tmp/gochopchop -COPY --from=build /app/chopchop.yml /tmp/chopchop.yml -WORKDIR /tmp -ENTRYPOINT ["/tmp/gochopchop"] \ No newline at end of file +COPY --from=builder /go/bin/gochopchop /bin/gochopchop +COPY chopchop.yml /etc/chopchop.yml +ENTRYPOINT [ "/bin/gochopchop" ] diff --git a/README.md b/README.md index b9e6650..8d1f585 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -

+
+ +
[![Build Status](https://github.com/michelin/ChopChop/workflows/Build%20ChopChop/badge.svg)](https://github.com/michelin/ChopChop/actions) [![License](https://img.shields.io/badge/license-Apache-green.svg)](https://opensource.org/licenses/Apache-2.0) @@ -11,7 +13,9 @@ Its goal is to scan several endpoints and identify exposition of services/files/folders through the webroot. Checks/Signatures are declared in a config file (by default: `chopchop.yml`), fully configurable, and especially by developers. -

+
+ +
> "Chop chop" is a phrase rooted in Cantonese. "Chop chop" means "hurry" and suggests that something should be done now and **without delay**. @@ -33,10 +37,8 @@ Checks/Signatures are declared in a config file (by default: `chopchop.yml`), fu We tried to make the build process painless and hopefully, it should be as easy as: - ```bash -$ go mod download -$ go build . +go build -o gochopchop cmd/main.go ``` There should be a resulting `gochopchop` binary in the folder. @@ -46,7 +48,7 @@ There should be a resulting `gochopchop` binary in the folder. Thanks to [Github Container Registry](https://github.blog/2020-09-01-introducing-github-container-registry/), we are able to provide you some freshly-build Docker images! ``` -docker run ghcr.io/michelin/gochopchop scan https://foobar.com -v debug +docker run ghcr.io/michelin/gochopchop scan -v debug https://example.com ``` But if you prefer, you can also build it locally, see below: @@ -59,62 +61,83 @@ docker build -t gochopchop . ## Usage -We are continuously trying to make `goChopChop` as easy as possible. Scanning a host with this utility is as simple as : +We are continuously trying to make `gochopchop` as easy as possible. Scanning a host with this utility is as simple as: ```bash -$ ./gochopchop scan https://foobar.com +./gochopchop scan https://example.com ``` +Notice you can specify multiple URLs. + ### Using Docker ```bash -docker run gochopchop scan https://foobar.com +docker run gochopchop scan https://example.com +``` + +Notice by default the Docker image has the configuration file at +`/etc/chopchop.yml`, so you may add `-c /etc/chopchop.yml` to your +command. If so, run the following command. + +```bash +docker run gochopchop scan -c /etc/chopchop.yml https://example.com ``` #### Custom configuration file +Of course you can use your own configuration files, using the following. + ```bash -docker run -v ./:/app chopchop scan -c /app/chopchop.yml https://foobar.com +docker run -v $(pwd):/app gochopchop scan -c /app/chopchop.yml https://example.com ``` ## What's next The Golang rewrite took place a couple of months ago but there's so much to do, still. Here are some features we are planning to integrate : -[x] Threading for better performance -[x] Ability to specify the number of concurrent threads -[x] Colors and better formatting -[x] Ability to filter checks/signatures to search for -[x] Mock and unit tests -[x] Github CI -And much more! + - [ ] Improve logging + - [ ] HTTP & SOCKS5 proxies + - [ ] Plugin method (GET, POST, ...) + - [ ] Plugin Cookie + - [ ] Improve caching (Docker build & CI) + - [ ] Implement a request gateway to avoid brute-forcing websites + - [ ] Re-implement `query_string` for the HTTP GET method + - [ ] Improve "severity reached" cases (currently ChopChop crashes if matches a plugin) + - [ ] Fix default status_code (200 if not specified) ## Testing -To quickly end-to-end test chopchop, we provided a web-server in `tests/server.go`. -To try it, please run `go run tests/server.go` then run chopchop with the following command `./gochopchop scan http://localhost:8000 --verbosity Debug`. -ChopChop should print "no vulnerabilities found". +### Unit tests + +Unit tests are achieved using Go-tests. You can run them using the following. + +```bash +go test ./... -cover +``` + +To visualize the code coverage, for developement purposes, you can also run. + +```bash +go test ./... -coverprofile=cov.out -count=1 && go tool cover -html=cov.out && rm cov.out +``` + +### Acceptance tests -There are also unit test that you can launch with `go test -v ./...`. -These tests are integrated in the github CI workflow. +For acceptance tests, those are achieved using RobotFramework. Notice you can +build ChopChop using another language and validate the CLI using those tests. +Run them using the following. + +```bash +cd robot +./run.sh +``` ## Available flags -You can find the available flags available for the `scan` command : - -| Flag | Full flag | Description | -|---|---|---| -| `-h` | `--help` | Help wizard | -| `-v` | `--verbosity` | Verbose level of logging | -| `-c` | `--signature` | Path of custom signature file | -| `-k` | `--insecure` | Disable SSL Verification | -| `-u` | `--url-file` | Path to a specified file containing urls to test | -| `-b` | `--max-severity` | Block the CI pipeline if severity is over or equal specified flag | -| `-e` | `--export` | Export type of the output (csv and/or json) | -|| `--export-filename` | Specify the filename for the export file(s) | -| `-t` | `--timeout` | Timeout for the HTTP requests | -|| `--severity-filter` | Filter Plugins by severity | -|| `--plugin-filter` | Filter Plugins by name of plugin | -|| `--threads` | Number of concurrent threads | +You can find the available flags and doc for each command using `gochopchop [cmd] -h`. + +Available commands are: + - `scan` to scan for endpoints ; + - `plugins` to parse and check the configuration file. ## Advanced usage @@ -124,61 +147,49 @@ Note: Redirectors like `>` for post processing can be used. - Ability to scan and disable SSL verification ```bash -$ ./gochopchop scan https://foobar.com --insecure +./gochopchop scan --insecure https://foobar.com ``` - Ability to scan with a custom configuration file (including custom plugins) ```bash -$ ./gochopchop scan https://foobar.com --insecure --signature test_config.yml +./gochopchop scan --insecure --signature test_config.yml https://foobar.com ``` -- Ability to list all the plugins or by severity : `plugins` or ` plugins --severity High` +- Ability to specify number of concurrent threads (in Go those are goroutines): `--threads 4` for 4 workers ```bash -$ ./gochopchop plugins --severity High +./gochopchop scan --threads 4 https://foobar.com ``` -- Ability to specify number of concurrent threads : `--threads 4` for 4 workers +- Ability to specify specific signatures to be checked, with a debug log level ```bash -$ ./gochopchop plugins --threads 4 +./gochopchop scan --timeout=1 --verbosity=debug --export=csv --export=json --export-filename=boo --plugin-filters=Git,Zimbra,Jenkins https://foobar.com ``` -- Ability to block the CI pipeline by severity level (equal or over specified severity) : `--max-severity Medium` +- Set a list or URLs located in a file ```bash -$ ./gochopchop scan https://foobar.com --max-severity Medium +./gochopchop scan --url-file url_file.txt ``` -- Ability to specify specific signatures to be checked +- Export GoChopChop results in CSV and JSON format ```bash -./gochopchop scan https://foobar.com --timeout 1 --verbosity --export=csv,json --export-filename boo --plugin-filters=Git,Zimbra,Jenkins +./gochopchop scan https://foobar.com --export csv --export json --export-filename results ``` - Ability to list all the plugins ```bash -$ ./gochopchop plugins -``` - -- List High severity plugins - -```bash -$ ./gochopchop plugins --severity High -``` - -- Set a list or URLs located in a file - -```bash -$ ./gochopchop scan --url-file url_file.txt +./gochopchop plugins ``` -- Export GoChopChop results in CSV and JSON format +- Ability to list all the plugins or by severity : `plugins` or `plugins --severity High` ```bash -$ ./gochopchop scan https://foobar.com --export=csv,json --export-filename results +./gochopchop plugins --severity High ``` ## Creating a new check @@ -186,17 +197,20 @@ $ ./gochopchop scan https://foobar.com --export=csv,json --export-filename resu Writing a new check is as simple as : ```yaml - - endpoint: "/.git/config" + - endpoints: + - "/.git/config" checks: - name: Git exposed match: - "[branch" remediation: Do not deploy .git folder on production servers description: Verifies that the GIT repository is accessible from the site - severity: "High" + severity: High ``` -An endpoint (eg. ```/.git/config```) is mapped to multiple checks which avoids sending X requests for X checks. Multiple checks can be done through a single HTTP request. +An endpoint (e.g. `/.git/config`) is mapped to multiple checks which avoids +sending X requests for X checks. Multiple checks are achieved through a +single HTTP request. Each check needs those fields: | Attribute | Type | Description | Optional ? | Example | @@ -210,21 +224,19 @@ Each check needs those fields: | no_headers | List of string | List of headers there should NOT be in the HTTP response | Yes | N/A | | match | List of string| List the strings there should be in the HTTP response | Yes | "[branch" | | no_match | List of string | List the strings there should NOT be in the HTTP response | Yes | N/A | -| query_string | GET parameters that have to be passed to the endpoint | String | Yes | `query_string: "id=FOO-chopchoptest"` | ## External Libraries -| Library Name | Link | License | -|---|---|---| -| Viper | https://github.com/spf13/viper | MIT License | -| Go-pretty | https://github.com/jedib0t/go-pretty| MIT License | -| Cobra | https://github.com/spf13/cobra| Apache License 2.0 | -| strfmt |https://github.com/go-openapi/strfmt | Apache License 2.0 | -| Go-homedir | https://github.com/mitchellh/go-homedir| MIT License | -| pkg-errors | https://github.com/pkg/errors| BSD 2 (Simplified License)| -| Go-runewidth | https://github.com/mattn/go-runewidth | MIT License | - -Please, refer to the `third-party.txt` file for further information. +| Library Name | Link | License | +|--------------|---------------------------------------|----------------------| +| go-md2man | https://github.com/cpuguy83/go-md2man | MIT License | +| strfmt | https://github.com/go-openapi/strfmt | Apache License 2.0 | +| go-cmp | https://github.com/google/go-cmp | BSD-3-Clause License | +| go-pretty | https://github.com/jedib0t/go-pretty | MIT License | +| go-runewidth | https://github.com/mattn/go-runewidth | MIT License | +| logrus | https://github.com/sirupsen/logrus | MIT License | +| cli | https://github.com/urfave/cli/v2 | MIT License | +| yaml | https://github.com/go-yaml/yaml | Apache License 2.0 | ## Talks diff --git a/chopchop.yml b/chopchop.yml index 41b52f9..b23fec9 100644 --- a/chopchop.yml +++ b/chopchop.yml @@ -1,15 +1,15 @@ ---- -insecure: false plugins: - - endpoint: "/status.shtml" + - endpoints: + - "/status.shtml" checks: - name: GENEREX UPS match: - 'UPS Status:' remediation: Make sure that GENEREX UPS access is restricted & monitored description: GENEREX UPS is accessible | don't move this rule to avoid client timeout - severity: "Medium" - - endpoint: "/" + severity: Medium + - endpoints: + - "/" checks: - name : GLPI vulnerable version match: @@ -20,129 +20,129 @@ plugins: remediation: Upgrade GLPI in latest version description: GLPI vulnerable version detected status_code: 200 - severity: "High" + severity: High - name : PACS NGI GXD5 match: - 'GXD5 Pacs Connexion utilisateur' remediation: Make sure that PACS NGI GXD5 access is restricted & monitored description: PACS NGI GXD5 detected status_code: 200 - severity: "High" + severity: High - name: AudioCodes SIP Gateway match: - 'AudioCodes' - '

Web Login

' remediation: Make sure that AudioCodes SIP Gateway access is restricted & monitored description: AudioCodes SIP Gateway detected - severity: "Informational" + severity: Informational - name: HP Printer headers: - "Server:Virata-EmWeb/R6_2_1" remediation: Make sure that HP Printer access is restricted & monitored description: HP Printer is accessible status_code: 200 - severity: "Low" + severity: Low - name: Printer (Lexmark, Dell, Toshiba, Sindoh) headers: - "Server:Lexmark_Web_Server" remediation: Make sure that Printer access is restricted & monitored description: Printer (Lexmark, Dell, Toshiba, Sindoh) is accessible status_code: 200 - severity: "Low" + severity: Low - name: Microsoft-IIS/7.0 - Windows Server 2003/2008 headers: - "Server:Microsoft-IIS/7.0" remediation: Upgrade to maintened version description: Microsoft-IIS/7.0 - Windows Server 2003/2008 - severity: "Informational" + severity: Informational - name: Microsoft-IIS/7.5 - Windows Server 2003/2008 headers: - "Server:Microsoft-IIS/7.5" remediation: Upgrade to maintened version description: Microsoft-IIS/7.5 - Windows Server 2003/2008 - severity: "Informational" + severity: Informational - name: GE ViewPoint match: - 'ViewPoint System Status' remediation: Make sure that GE ViewPoint System Status access is restricted & monitored description: GE ViewPoint System Status is accessible / sensitive information leaking status_code: 200 - severity: "Low" + severity: Low - name: Ascom IP-DECT Base Station match: - '<select product="Ascom IP-DECT Base Station"' remediation: Make sure that Ascom IP-DECT Base Station access is restricted & monitored description: Ascom IP-DECT Base Station is accessible status_code: 200 - severity: "Informational" + severity: Informational - name: EMC Unisphere match: - 'Unisphere<br>' remediation: Make sure that EMC Unisphere access is restricted & monitored description: EMC Unisphere is accessible status_code: 200 - severity: "Low" + severity: Low - name: F-Secure Policy Manager Server match: - '<title>F-Secure Policy Manager Server' remediation: Make sure that F-Secure Policy Manager Server access is monitored description: F-Secure Policy Manager Server is accessible status_code: 200 - severity: "Informational" + severity: Informational - name: Apache2 Debian Default Page match: - 'Apache2 Debian Default Page: It works' remediation: Remove the symbolic link from the Apache default configuration description: Detects the presence of a default Apache page - severity: "Informational" + severity: Informational - name: Cisco IOS headers: - "Server:cisco-IOS" remediation: Make sure that Cisco IOS access is restricted & monitored description: Cisco IOS is accessible - severity: "Low" + severity: Low - name: Odin match: - '

' remediation: Make sure that Odin service automation access is restricted & monitored description: Odin service automation is accessible - severity: "Informational" + severity: Informational - name: Nordex Control headers: - "Server:Jetty/3.1.8 (Windows 2000 5.0 x86)" remediation: Make sure that Nordex Control access is restricted & monitored description: Nordex Control is accessible - severity: "Low" + severity: Low - name: EIG GaugeTech Electricity Meter headers: - "Server:EIG Embedded Web Server" remediation: Make sure that EIG GaugeTech Electricity Meter access is restricted & monitored description: EIG GaugeTech Electricity Meter is accessible - severity: "Low" + severity: Low - name: Weave Scope match: - 'Weave Scope' remediation: Make sure that Weave Scope access is restricted & monitored description: Weave Scope is accessible - severity: "Medium" + severity: Medium - name: NETAVIS Observer match: - 'NETAVIS Observer' remediation: Make sure that NETAVIS Observer access is restricted & monitored description: NETAVIS Observer is accessible - severity: "Informational" + severity: Informational - name: Jenkins match: - "hudson" remediation: Monitor access to your jenkins instance. description: Checks that the domain is not a Jenkins instance - severity: "Informational" + severity: Informational headers: - "Cache-Control:no-cache,no-store,must-revalidate" - name: BigIPServer remediation: Encrypt sticky cookie to avoid leaking internal IPs description: Detects the presence of unencrypted sticky cookies that allow to retrieve internal Ips - severity: "Medium" + severity: Medium headers: - "Set-Cookie:BIGipServer" - name: TakeOver @@ -170,19 +170,19 @@ plugins: - ".asmx" remediation: Monitor that webservices are well monitored. description: Checks the presence of webservices in the page - severity: "Informational" + severity: Informational - name: Gitlab instance match: - "GitLab" remediation: Make sure that access to Gitlab is properly monitored description: Checks if a Gitlab instance exists - severity: "Low" + severity: Low - name: Apache2 Ubuntu Default Page match: - "Apache2 Ubuntu Default Page" remediation: Remove the symbolic link from the Apache default configuration description: Detects the presence of a default Apache page - severity: "Informational" + severity: Informational - name: Drupal CMS match: - "drupal" @@ -190,62 +190,63 @@ plugins: - '"core/' remediation: Check that the version is the last one available on the vendor's website description: Get the Drupal version of the site - severity: "Low" + severity: Low - name: Status Code 500 status_code: 500 remediation: Check that the server has not completely fallen into error description: Check return code 500 - severity: "Low" + severity: Low - name: Iis headers: - "Server:Microsoft-IIS/6.0" remediation: Patch the server as soon as possible description: Checks that the server is an IIS 6.0 - severity: "Informational" + severity: Informational - name: Indexof match: - "Index of" remediation: Implementing rules at the application server level to prevent directory listing description: Checks that the domain root does not return a file/folder list - severity: "Low" + severity: Low - name: IndexOf2 match: - "<dir>" remediation: Implementing rules at the application server level to prevent directory listing description: Checks that the domain root does not return a file/folder list (simple encoding) - severity: "Low" + severity: Low - name: MySQLError match: - 'You have an error in your SQL syntax' remediation: Do not display MySQL errors on web pages description: Checks that MySQL errors are not displayed - severity: "Medium" + severity: Medium - name: NginxDefaultPage match: - 'Welcome to nginx!' remediation: Delete symbolic link from Nginx default configuration description: Checks that the default Nginx site is not accessible - severity: "Low" + severity: Low - name: Osticket match: - 'Helpdesk software - powered by osTicket' remediation: Check that the passwords used are robust description: Checks that the domain is not an OS Ticket instance - severity: "Informational" + severity: Informational - name: PHP open code match: - '' remediation: Delete wildcards from xml files description: Checks for the presence of a crossdomain.xml file with a wildcard for the domain - severity: "High" - - endpoint: "/manager/html" + severity: High + - endpoints: + - "/manager/html" checks: - name: tomcat manager status_code: 401 remediation: Disable this interface in production description: Checks that under /manager/html the Tomcat administration interface is not accessible - severity: "Medium" - - endpoint: "/.htpasswd" + severity: Medium + - endpoints: + - "/.htpasswd" checks: - name: .htpasswd not interpreted match: - ":" remediation: Delete file and reset leaky passwords description: Checks for the presence of an .htpasswd file at the root of the domain - severity: "Medium" + severity: Medium status_code: 200 no_match: - "' remediation: Check that the administration interfaces are well protected description: Detects the presence of a login page using the Apostrophe Framework (from Digital Factory) - severity: "Informational" + severity: Informational - name: Grafana match: - "isGrafanaAdmin" remediation: Check that the passwords used are robust description: Check access to Grafana administration - severity: "Informational" - - endpoint: "/user/login" + severity: Informational + - endpoints: + - "/user/login" checks: - name: eZ Publish Admin Panel match: - "Log in to the Administration Interface of eZ Publish" remediation: Check that the passwords used are robust description: Check access to the eZ Publish administration - severity: "Low" - - endpoint: "/fckeditor/editor/filemanager/browser/default/browser.html" + severity: Low + - endpoints: + - "/fckeditor/editor/filemanager/browser/default/browser.html" checks: - name: FckEditor match: - "Resources Browser" remediation: Put authentication on this form description: Check access to a wysiwyg fckeditor - severity: "High" - - endpoint: "/.idea/workspace.xml" + severity: High + - endpoints: + - "/.idea/workspace.xml" checks: - name: Idea WorkSpace match: - "' remediation: Check that no sensitive information is present in the web.config description: Checks that the web.config configuration file of the ASP.net server is not accessible - severity: "Low" - - endpoint: "/wp-login.php" + severity: Low + - endpoints: + - "/wp-login.php" checks: - name: Wordpress Login Page all_match: @@ -470,24 +495,27 @@ plugins: - 'BIG-IP" remediation: Make sure that F5 BIG-IP - TMUI access is monitored description: Checks that under /tmui a F5 BIG-IP TMUI is not accessible - severity: "Low" - - endpoint: "/images/imgpaper.png" + severity: Low + - endpoints: + - "/images/imgpaper.png" checks: - name: Possible Trickbot Trojan Payload hosting imgpaper.png on Apache headers: @@ -512,15 +542,16 @@ plugins: remediation: Make sure your system isn't compromised description: Possible Trickbot Trojan Payload hosting in /images/imgpaper.png status_code: 200 - severity: "High" + severity: High - name: Trickbot Trojan Payload hosting imgpaper.png on Nginx headers: - 'Content-Type:application/octet-stream' remediation: Make sure your system isn't compromised description: Possible Trickbot Trojan Payload hosting in /images/imgpaper.png status_code: 200 - severity: "High" - - endpoint: "/images/cursor.png" + severity: High + - endpoints: + - "/images/cursor.png" checks: - name: Possible Trickbot Trojan Payload hosting cursor.png on Apache headers: @@ -528,15 +559,16 @@ plugins: remediation: Make sure your system isn't compromised description: Possible Trickbot Trojan Payload hosting in /images/cursor.png status_code: 200 - severity: "High" + severity: High - name: Trickbot Trojan Payload hosting cursor.png on Nginx headers: - 'Content-Type:application/octet-stream' remediation: Make sure your system isn't compromised description: Possible Trickbot Trojan Payload hosting in /images/cursor.png status_code: 200 - severity: "High" - - endpoint: "/images/redcar.png" + severity: High + - endpoints: + - "/images/redcar.png" checks: - name: Possible Trickbot Trojan Payload hosting redcar.png on Apache headers: @@ -544,15 +576,16 @@ plugins: remediation: Make sure your system isn't compromised description: Possible Trickbot Trojan Payload hosting in /images/redcar.png status_code: 200 - severity: "High" + severity: High - name: Trickbot Trojan Payload hosting redcar.png on Nginx headers: - 'Content-Type:application/octet-stream' remediation: Make sure your system isn't compromised description: Possible Trickbot Trojan Payload hosting in /images/redcar.png status_code: 200 - severity: "High" - - endpoint: "/ico/VidT6cErs" + severity: High + - endpoints: + - "/ico/VidT6cErs" checks: - name: Possible Trickbot Trojan Payload hosting VidT6cErs no_match: @@ -563,8 +596,9 @@ plugins: remediation: Make sure your system isn't compromised description: Possible Trickbot Trojan Payload hosting in /ico/VidT6cErs status_code: 200 - severity: "High" - - endpoint: "/admin/libs/prettify-4-Mar-2013/prettify.css" + severity: High + - endpoints: + - "/admin/libs/prettify-4-Mar-2013/prettify.css" checks: - name: Stormshield SNS Web Admin Console headers: @@ -572,8 +606,9 @@ plugins: remediation: Make sure that Stormshield SNS Web Admin Console access is restricted & monitored description: Stormshield SNS Web Admin Console is accessible status_code: 200 - severity: "Low" - - endpoint: "/auth" + severity: Low + - endpoints: + - "/auth" checks: - name: Stormshield Web Portal match: @@ -582,8 +617,9 @@ plugins: remediation: Make sure that Stormshield Web Portal access is restricted & monitored description: Stormshield Web Portal is accessible status_code: 200 - severity: "Informational" - - endpoint: "/ui" + severity: Informational + - endpoints: + - "/ui" checks: - name: VMware ESXi match: @@ -591,8 +627,9 @@ plugins: remediation: Make sure that VMware ESXi access is restricted & monitored description: VMware ESXi is accessible status_code: 200 - severity: "Low" - - endpoint: "/vsphere-client" + severity: Low + - endpoints: + - "/vsphere-client" checks: - name: VMware vCenter match: @@ -600,8 +637,9 @@ plugins: remediation: Make sure that VMware vCenter access is restricted & monitored description: VMware vCenter is accessible status_code: 200 - severity: "Low" - - endpoint: "/eai/index.html" + severity: Low + - endpoints: + - "/eai/index.html" checks: - name: Enovacom Suite V2 match: @@ -609,8 +647,9 @@ plugins: remediation: Make sure that EAI Enovacom Suite V2 access is restricted & monitored description: EAI Enovacom Suite V2 is accessible status_code: 200 - severity: "Low" - - endpoint: "/mailscanner/login.php" + severity: Low + - endpoints: + - "/mailscanner/login.php" checks: - name: MailWatch match: @@ -618,8 +657,9 @@ plugins: remediation: Make sure that MailWatch access is monitored description: MailWatch is accessible status_code: 200 - severity: "Low" - - endpoint: "/fog/management/index.php" + severity: Low + - endpoints: + - "/fog/management/index.php" checks: - name: FOG Project match: @@ -628,8 +668,9 @@ plugins: remediation: Make sure that FOG Project access is monitored description: FOG Project is accessible status_code: 200 - severity: "Low" - - endpoint: "/.well-known/security.txt" + severity: Low + - endpoints: + - "/.well-known/security.txt" checks: - name: Security.txt match: @@ -637,24 +678,27 @@ plugins: remediation: Great ! A Security.txt file for contact is present description: Detects the presence of Security.txt file status_code: 200 - severity: "Informational" - - endpoint: "/XsEXPL" + severity: Informational + - endpoints: + - "/XsEXPL" checks: - name: Xplore Web RIS match: - 'Xplore Exploitation' remediation: Make sure that Xplore Web RIS access is restricted & monitored description: Xplore Web RIS is accessible - severity: "Informational" - - endpoint: "/zimbraAdmin" + severity: Informational + - endpoints: + - "/zimbraAdmin" checks: - name: Zimbra Administration match: - 'Zimbra Collaboration Suite Web Client' remediation: Make sure that Zimbra Administration access is restricted & monitored description: Zimbra Administration is accessible - severity: "Low" - - endpoint: "/public/img/mongo-express-logo.png" + severity: Low + - endpoints: + - "/public/img/mongo-express-logo.png" checks: - name: Mongo Express headers: @@ -662,8 +706,9 @@ plugins: remediation: Make sure that Mongo Express access is restricted & monitored description: Mongo Express is accessible status_code: 200 - severity: "High" - - endpoint: "/login.html" + severity: High + - endpoints: + - "/login.html" checks: - name: Polycom headers: @@ -673,8 +718,9 @@ plugins: remediation: Make sure that Polycom access is restricted & monitored description: Polycom Video Conferencing is accessible status_code: 200 - severity: "Informational" - - endpoint: "/securityRealm/user/admin/search/index?q=a" + severity: Informational + - endpoints: + - "/securityRealm/user/admin/search/index?q=a" checks: - name: Jenkins CVE-2018-1000861 (RCE) match: @@ -684,8 +730,9 @@ plugins: - 'HTTP ERROR 404 Not Found' remediation: Patch the server as soon as possible description: Jenkins server is vulnerable to RCE CVE-2018-1000861 - severity: "High" - - endpoint: "/?MAIN=TOPACCESS" + severity: High + - endpoints: + - "/?MAIN=TOPACCESS" checks: - name: TopAccess Toshiba MFP match: @@ -693,8 +740,9 @@ plugins: remediation: Make sure that TopAccess access is restricted & monitored description: TopAccess Toshiba MFP is accessible status_code: 200 - severity: "Low" - - endpoint: "/ePrint/ePrintConfigDyn.xml" + severity: Low + - endpoints: + - "/ePrint/ePrintConfigDyn.xml" checks: - name: HP Printer headers: @@ -702,8 +750,9 @@ plugins: remediation: Make sure that HP Printer access is restricted & monitored description: HP Printer is accessible status_code: 200 - severity: "Low" - - endpoint: "/config.html" + severity: Low + - endpoints: + - "/config.html" checks: - name: Zebra Label Printer match: @@ -711,8 +760,9 @@ plugins: remediation: Make sure that Zebra Label Printer access is restricted & monitored description: Zebra Label Printer is accessible status_code: 200 - severity: "Low" - - endpoint: "/spip.php?page=login" + severity: Low + - endpoints: + - "/spip.php?page=login" checks: - name : SPIP admin interface match: @@ -720,7 +770,7 @@ plugins: remediation: Make sure that SPIP admin interface access is restricted & monitored description: SPIP admin interface is accessible status_code: 200 - severity: "Informational" + severity: Informational - name : SPIP vulnerable version match: - 'content="SPIP' @@ -732,8 +782,9 @@ plugins: remediation: Upgrade SPIP in latest version description: SPIP vulnerable version detected status_code: 200 - severity: "High" - - endpoint: "/support/support.php" + severity: High + - endpoints: + - "/support/support.php" checks: - name : Xerox Printer headers: @@ -743,18 +794,19 @@ plugins: remediation: Make sure that Xerox Printer access is restricted & monitored description: Xerox Printer is accessible status_code: 200 - severity: "Low" - - endpoint: "/cgi-bin/dynamic/topbar.html" + severity: Low + - endpoints: + - "/cgi-bin/dynamic/topbar.html" checks: - name : Lexmark Printer match: - 'Lexmark' - remediation: Make sure that Lexmark Printer access is restricted & monitored description: Lexmark Printer is accessible status_code: 200 - severity: "Low" - - endpoint: "/felia/user/signin?source=" + severity: Low + - endpoints: + - "/felia/user/signin?source=" checks: - name : Aklia Lisis - traçabilité patients match: @@ -764,4 +816,4 @@ plugins: remediation: Make sure that Aklia Lisis access is restricted & monitored description: Aklia Lisis is accessible status_code: 200 - severity: "Low" + severity: Low diff --git a/cmd/list.go b/cmd/list.go deleted file mode 100644 index da0f059..0000000 --- a/cmd/list.go +++ /dev/null @@ -1,67 +0,0 @@ -package cmd - -import ( - "fmt" - "gochopchop/core" - "os" - - "github.com/jedib0t/go-pretty/table" - "github.com/spf13/cobra" -) - -type listOptions struct { - Severity string -} - -func init() { - pluginCmd := &cobra.Command{ - Use: "plugins", - Short: "list checks of configuration file", - RunE: runList, - } - addSignaturesFlag(pluginCmd) - pluginCmd.Flags().StringP("severity", "s", "", "severity option for list tag") // --severity ou -s - - rootCmd.AddCommand(pluginCmd) -} - -func runList(cmd *cobra.Command, args []string) error { - signatures, err := parseSignatures(cmd) - if err != nil { - return err - } - options, err := parseOptions(cmd) - if err != nil { - return err - } - cpt := 0 - t := table.NewWriter() - t.SetOutputMirror(os.Stdout) - t.AppendHeader(table.Row{"URL", "Plugin Name", "Severity", "Description"}) - for _, plugin := range signatures.Plugins { - for _, check := range plugin.Checks { - if options.Severity == "" || options.Severity == string(check.Severity) { - t.AppendRow([]interface{}{plugin.Endpoint, check.Name, check.Severity, check.Description}) - cpt++ - } - } - } - t.AppendFooter(table.Row{"", "", "Total Checks", cpt}) - t.Render() - return nil -} - -func parseOptions(cmd *cobra.Command) (*listOptions, error) { - options := new(listOptions) - severity, err := cmd.Flags().GetString("severity") - if err != nil { - return nil, fmt.Errorf("invalid value for severity: %v", err) - } - if severity != "" { - if !core.ValidSeverity(severity) { - return nil, fmt.Errorf("Invalid severity level : %s. Please use : %s", severity, core.SeveritiesAsString()) - } - options.Severity = severity - } - return options, nil -} diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..446daf4 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,306 @@ +package main + +import ( + "context" + "io" + "os" + "os/signal" + "sort" + "syscall" + + "github.com/michelin/gochopchop/internal" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +const ( + cliLogo = ` + ________ _________ .__ _________ .__ ._. + / _____/ ____ \_ ___ \| |__ ____ ______ \_ ___ \| |__ ____ ______ | | +/ \ ___ / _ \ ______ / \ \/| | \ / _ \\____ \/ \ \/| | \ / _ \\____ \ | | +\ \_\ ( <_> ) /_____/ \ \___| Y ( <_> ) |_> > \___| Y ( <_> ) |_> > \| + \______ /\____/ \______ /___| /\____/| __/ \______ /___| /\____/| __/ __ + \/ \/ \/ |__| \/ \/ |__| \/ +` + AppHelpTemplate = cliLogo + ` +{{.Name}}{{if .Usage}} - {{.Usage}}{{end}} + +Usage: + chopchop [command]{{"\n"}} + +{{- if .Description}} +DESCRIPTION: + {{.Description | nindent 3 | trim}}{{end}} + +{{- if .VisibleCommands}} +Available Commands:{{range .VisibleCategories}}{{if .Name}} + {{.Name}}:{{range .VisibleCommands}} + {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{else}}{{range .VisibleCommands}} + {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{end}}{{if .VisibleFlags}} + +Flags: + {{range $index, $option := .VisibleFlags}}{{if $index}} + {{end}}{{$option}}{{end}}{{end}} + +Use "chopchop [command] --help" for more information about a command. +` + CommandHelpTemplate = `{{.Usage}} + +Usage: + {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .VisibleFlags}} [flags]{{end}}{{end}}{{if .VisibleFlags}} + +Flags: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +` +) + +func flagsMdw(flags []cli.Flag) []cli.Flag { + // Build shared flags + f := []cli.Flag{ + &cli.IntFlag{ + Name: "threads", + Usage: "number of threads (goroutines to be exact)", + Value: 1, + }, + &cli.StringFlag{ + Name: "verbosity", + Aliases: []string{"v"}, + Usage: "log level (debug, info, warn, error, fatal, panic)", + Value: "warning", + }, + } + + return append(flags, f...) +} + +func cliMdw(f func(*cli.Context) error) func(*cli.Context) error { + return func(c *cli.Context) error { + // Setup logs + logrus.SetFormatter(&logrus.JSONFormatter{}) + logrus.SetOutput(os.Stdout) + lvl, err := logrus.ParseLevel(c.String("verbosity")) + if err != nil { + return err + } + logrus.SetLevel(lvl) + logrus.Debug("verbosity:", lvl) + + // Call the wrapped cli func + return f(c) + } +} + +func main() { + app := &cli.App{ + Name: "ChopChop", + Usage: "CLI tool to help developers scanning endpoints and identifying exposition of sensitive services/files/folders.\nhttps://github.com/michelin/ChopChop.", + Commands: []*cli.Command{ + { + Name: "plugins", + Usage: "list checks of configuration file", + Action: cliMdw(cmdPlugins), + Flags: flagsMdw([]cli.Flag{ + &cli.StringFlag{ + Name: "severity", + Aliases: []string{"s"}, + Usage: "severity option for list tag", + Value: "Informational", + }, + &cli.StringFlag{ + Name: "signatures", + Aliases: []string{"c"}, + Usage: "path to signature file", + Value: "chopchop.yml", + }, + }), + }, { + Name: "scan", + Usage: "scan endpoints to check if services/files/folders are exposed", + Action: cliMdw(cmdScan), + Flags: flagsMdw([]cli.Flag{ + &cli.StringSliceFlag{ + Name: "export", + Aliases: []string{"e"}, + Usage: "export of the output (" + internal.ExportersList() + ")", + Value: &cli.StringSlice{}, + }, + &cli.StringFlag{ + Name: "export-filename", + Usage: "filename for export files", + Value: "", + }, + &cli.BoolFlag{ + Name: "insecure", + Aliases: []string{"k"}, + Usage: "check SSL certificate", + Value: false, + }, + &cli.StringFlag{ + Name: "max-severity", + Aliases: []string{"b"}, + Usage: "block the CI pipeline if severity is over or equal specified flag", + Value: "Informational", + }, + &cli.StringSliceFlag{ + Name: "plugin-filters", + Usage: "filter by the name of the plugin (engine will only check for plugin with the same name)", + Value: &cli.StringSlice{}, + }, + &cli.StringFlag{ + Name: "severity-filter", + Usage: "filter by severity (engine will check for same severity checks)", + Value: "Informational", + }, + &cli.StringFlag{ + Name: "signatures", + Aliases: []string{"c"}, + Usage: "path to signature file", + Value: "chopchop.yml", + }, + &cli.IntFlag{ + Name: "timeout", + Aliases: []string{"t"}, + Usage: "timeout (in s) for the HTTP requests", + Value: 10, + }, + &cli.StringFlag{ + Name: "url-file", + Aliases: []string{"u"}, + Usage: "path to a specified file containing urls to test", + Value: "", + }, + }), + }, + }, + } + cli.AppHelpTemplate = AppHelpTemplate + cli.CommandHelpTemplate = CommandHelpTemplate + + // Setup stop signals + ctx, cancel := context.WithCancel(context.Background()) + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + defer func() { + signal.Stop(sigs) + cancel() // Triggers the <-ctx.Done() in the following goroutine + }() + go func() { + select { + case <-sigs: + logrus.Warn("Keyboard interrupt detected.") + cancel() + os.Exit(1) + case <-ctx.Done(): + return + } + }() + + err := app.RunContext(ctx, os.Args) + if err != nil { + logrus.Fatal(err) + } +} + +func cmdScan(c *cli.Context) error { + // Build the config + insecure := c.Bool("insecure") + exprt := c.StringSlice("export") + pluginFilters := c.StringSlice("plugin-filters") + exportFilename := c.String("export-filename") + maxSeverity := c.String("max-severity") + severityFilter := c.String("severity-filter") + urlFile := c.String("url-file") + timeout := c.Int("timeout") + threads := c.Int("threads") + args := c.Args() + + logrus.Debug("insecure:", insecure) + logrus.Debug("export:", exprt) + logrus.Debug("plugin-filters:", pluginFilters) + logrus.Debug("export-filename:", exportFilename) + logrus.Debug("max-severity:", maxSeverity) + logrus.Debug("severity-filter:", severityFilter) + logrus.Debug("url-file:", urlFile) + logrus.Debug("timeout:", timeout) + logrus.Debug("threads:", threads) + logrus.Debug("args:", args) + + var urlFileReader io.Reader + if urlFile != "" { + var err error + urlFileReader, err = os.Open(urlFile) + if err != nil { + return err + } + } + + config, err := internal.BuildConfig(insecure, exprt, pluginFilters, exportFilename, maxSeverity, severityFilter, urlFileReader, threads, timeout, args.Slice()) + if err != nil { + return err + } + + // Parse signatures + signatures := c.String("signatures") + + signFile, err := internal.ReaderFromFile(signatures) + if err != nil { + return err + } + sign, err := internal.ParseSignatures(signFile) + if err != nil { + return err + } + + // Build the CoreScanner + scanner, err := internal.NewCoreScanner(config, sign) + if err != nil { + return err + } + + // Start the scan + results, dur, err := internal.Scan(scanner, config.Urls, c.Done()) + if err != nil { + return err + } + logrus.Info("Scan execution time: ", dur) + + // Sort and export the results + sort.Stable(results) + err = internal.ExportResults(results, config, exportFilename) + if err != nil { + return err + } + + return nil +} + +func cmdPlugins(c *cli.Context) error { + // Parse signatures + signatures := c.String("signatures") + + logrus.Debug("signatures:", signatures) + + signFile, err := internal.ReaderFromFile(signatures) + if err != nil { + return err + } + sign, err := internal.ParseSignatures(signFile) + if err != nil { + return err + } + + // Parse severity + severity := c.String("severity") + + sev, err := internal.StringToSeverity(severity) + if err != nil { + return err + } + sevStr, _ := sev.String() + + // Print signatures in stdout + internal.PrintSignatures(sign, sevStr, os.Stdout) + + return nil +} diff --git a/cmd/root.go b/cmd/root.go deleted file mode 100644 index 83403a5..0000000 --- a/cmd/root.go +++ /dev/null @@ -1,78 +0,0 @@ -package cmd - -import ( - "context" - "io" - "os" - "os/signal" - "syscall" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -func init() { - -} - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "chopchop", - Short: "tool for dynamic application security testing on web applications", - Long: ` - ________ _________ .__ _________ .__ ._. - / _____/ ____ \_ ___ \| |__ ____ ______ \_ ___ \| |__ ____ ______ | | -/ \ ___ / _ \ ______ / \ \/| | \ / _ \\____ \/ \ \/| | \ / _ \\____ \ | | -\ \_\ ( <_> ) /_____/ \ \___| Y ( <_> ) |_> > \___| Y ( <_> ) |_> > \| - \______ /\____/ \______ /___| /\____/| __/ \______ /___| /\____/| __/ __ - \/ \/ \/ |__| \/ \/ |__| \/ -Link: https://github.com/michelin/ChopChop`, - SilenceUsage: true, -} - -var v string - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { - if err := setupLogs(os.Stdout, v); err != nil { - return err - } - return nil - } - - rootCmd.PersistentFlags().StringVarP(&v, "verbosity", "v", log.WarnLevel.String(), "Log level (debug, info, warn, error, fatal, panic)") - rootCmd.PersistentFlags().IntP("threads", "", 1, "Number of threads") - ctx, cancel := context.WithCancel(context.Background()) - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - defer func() { - signal.Stop(sigs) - cancel() - }() - go func() { - select { - case <-sigs: - log.Warn("\n[!] Keyboard interrupt detected.") - cancel() - os.Exit(1) - case <-ctx.Done(): - } - }() - if err := rootCmd.ExecuteContext(ctx); err != nil { - log.Warn(err) - os.Exit(1) - } -} - -func setupLogs(out io.Writer, level string) error { - log.SetFormatter(&log.JSONFormatter{}) - log.SetOutput(out) - lvl, err := log.ParseLevel(level) - if err != nil { - return err - } - log.SetLevel(lvl) - return nil -} diff --git a/cmd/scan.go b/cmd/scan.go deleted file mode 100644 index eee9043..0000000 --- a/cmd/scan.go +++ /dev/null @@ -1,228 +0,0 @@ -package cmd - -import ( - "bufio" - "fmt" - "gochopchop/core" - "gochopchop/internal/export" - "gochopchop/internal/formatting" - "gochopchop/internal/httpget" - "net/url" - "os" - "time" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -func init() { - scanCmd := &cobra.Command{ - Use: "scan", - Short: "scan endpoints to check if services/files/folders are exposed", - RunE: runScan, - } - addSignaturesFlag(scanCmd) - - scanCmd.Flags().BoolP("insecure", "k", false, "Check SSL certificate") // --insecure ou -n - scanCmd.Flags().StringP("url-file", "u", "", "path to a specified file containing urls to test") // --uri-file ou -f - scanCmd.Flags().StringP("max-severity", "b", "", "block the CI pipeline if severity is over or equal specified flag") // --max-severity ou -m - scanCmd.Flags().StringSliceP("export", "e", []string{}, "export of the output (csv and json)") //--export ou --e - scanCmd.Flags().StringP("export-filename", "", "", "filename for export files") // --export-filename - scanCmd.Flags().IntP("timeout", "t", 10, "Timeout for the HTTP requests (default: 10s)") // --timeout ou -ts - scanCmd.Flags().StringP("severity-filter", "", "", "Filter by severity (engine will check for same severity checks)") // --severity-filter - scanCmd.Flags().StringSliceP("plugin-filters", "", []string{}, "Filter by the name of the plugin (engine will only check for plugin with the same name)") // --plugin-filter - rootCmd.AddCommand(scanCmd) -} - -func runScan(cmd *cobra.Command, args []string) error { - config, err := parseConfig(cmd, args) - if err != nil { - return err - } - - signatures, err := parseSignatures(cmd) - if err != nil { - return err - } - - begin := time.Now() - - fetcher := httpget.NewFetcher(config.HTTP.Insecure, config.HTTP.Timeout) - noRedirectFetcher := httpget.NewNoRedirectFetcher(config.HTTP.Insecure, config.HTTP.Timeout) - - scanner := core.NewScanner(fetcher, noRedirectFetcher, signatures, config.Threads) - - result, err := scanner.Scan(cmd.Context(), config.Urls) - if err != nil { - return err - } - - log.Info("Scan execution time:", time.Since(begin)) - - if len(result) > 0 { - - formatting.PrintTable(result, os.Stdout) - - if contains(config.ExportFormats, "json") { - export.ExportJSON(config.ExportFilename, result) - } - if contains(config.ExportFormats, "csv") { - export.ExportCSV(config.ExportFilename, result) - } - - if config.MaxSeverity != "" { - for _, output := range result { - if core.SeverityReached(config.MaxSeverity, output.Severity) { - return fmt.Errorf("Max severity level reached, exiting with error code") - } - } - } - } else { - log.Info("No vulnerabilities found. Exiting...") - } - return nil -} - -func parseConfig(cmd *cobra.Command, args []string) (*core.Config, error) { - - urlFile, err := cmd.Flags().GetString("url-file") - if err != nil { - return nil, fmt.Errorf("invalid value for url-file: %v", err) - } - - if urlFile != "" && len(args) >= 1 { - // both urlFile and url are set, abort - return nil, fmt.Errorf("Can't specify url with url list flag") - } - if urlFile == "" && len(args) == 0 { - // no urlFile and no argument, abort - return nil, fmt.Errorf("No url provided, please set the input-file flag or provide an url as an argument") - } - - var urls []string - if urlFile != "" { - content, err := os.Open(urlFile) - if err != nil { - return nil, err - } - defer content.Close() - scanner := bufio.NewScanner(content) - for scanner.Scan() { - url := scanner.Text() - if !isURL(url) { - log.Warn("url: ", url, " - is not valid - skipping scan") - continue - } - urls = append(urls, url) - } - if err := scanner.Err(); err != nil { - return nil, err - } - } - - if len(args) > 1 { - return nil, fmt.Errorf("Please provide only one URL") - } - - if len(args) == 1 { - url := args[0] - if isURL(url) { - urls = append(urls, url) - } else { - return nil, fmt.Errorf("Please provide a valid URL") - } - } - - insecure, err := cmd.Flags().GetBool("insecure") - if err != nil { - return nil, fmt.Errorf("invalid value for insecure: %v", err) - } - - severityFilter, err := cmd.Flags().GetString("severity-filter") - if err != nil { - return nil, fmt.Errorf("invalid value for severity-filter: %v", err) - } - if severityFilter != "" { - if !core.ValidSeverity(severityFilter) { - return nil, fmt.Errorf("Invalid severity level : %s. Please use : %s", severityFilter, core.SeveritiesAsString()) - } - } - - pluginFilters, err := cmd.Flags().GetStringSlice("plugin-filters") - if err != nil { - return nil, fmt.Errorf("invalid value for plugin-filters: %v", err) - } - - exportFormats, err := cmd.Flags().GetStringSlice("export") - if err != nil { - return nil, fmt.Errorf("invalid value for export formats: %v", err) - } - if len(exportFormats) > 0 { - for _, f := range exportFormats { - if f != "csv" && f != "json" { - return nil, fmt.Errorf("invalid value for export: %v , expected csv or json", f) - } - } - } - - maxSeverity, err := cmd.Flags().GetString("max-severity") - if err != nil { - return nil, fmt.Errorf("invalid value for max sevirity : %v", err) - } - if maxSeverity != "" && !core.ValidSeverity(maxSeverity) { - return nil, fmt.Errorf("Invalid max severity level : %s. Please use : %s", maxSeverity, core.SeveritiesAsString()) - } - - exportFilename, err := cmd.Flags().GetString("export-filename") - if err != nil { - return nil, fmt.Errorf("invalid value for exportFilename: %v", err) - } - if exportFilename == "" { - now := time.Now().Format("2006-01-02_15-04-05") - exportFilename = fmt.Sprintf("gochopchop_%s", now) - } - - timeout, err := cmd.Flags().GetInt("timeout") - if err != nil { - return nil, fmt.Errorf("Invalid value for timeout: %v", err) - } - - threads, err := rootCmd.Flags().GetInt("threads") - if err != nil { - return nil, fmt.Errorf("invalid value for threads: %w", err) - } - - if threads <= 0 { - return nil, fmt.Errorf("The number of threads must be positive") - } - - config := &core.Config{ - HTTP: core.HTTPConfig{ - Insecure: insecure, - Timeout: timeout, - }, - MaxSeverity: maxSeverity, - ExportFormats: exportFormats, - Urls: urls, - ExportFilename: exportFilename, - SeverityFilter: severityFilter, - PluginFilter: pluginFilters, - Threads: threads, - } - - return config, nil -} - -func isURL(str string) bool { - u, err := url.Parse(str) - return err == nil && u.Scheme != "" && u.Host != "" -} - -func contains(s []string, e string) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} diff --git a/cmd/signatures.go b/cmd/signatures.go deleted file mode 100644 index a7cf29b..0000000 --- a/cmd/signatures.go +++ /dev/null @@ -1,89 +0,0 @@ -package cmd - -import ( - "fmt" - "gochopchop/core" - "io/ioutil" - "os" - "strings" - - "github.com/spf13/cobra" - "gopkg.in/yaml.v2" -) - -var signatureFlagName = "signatures" -var signatureFlagShorthand = "c" -var signatureDefaultFilename = "chopchop.yml" - -func addSignaturesFlag(cmd *cobra.Command) error { - cmd.Flags().StringP(signatureFlagName, signatureFlagShorthand, signatureDefaultFilename, "path to signature file") // --signature ou -c - return nil -} - -func parseSignatures(cmd *cobra.Command) (*core.Signatures, error) { - - signatureFile, err := cmd.Flags().GetString(signatureFlagName) - if err != nil { - return nil, fmt.Errorf("Invalid value for signatureFile: %v", err) - } - if _, err := os.Stat(signatureFile); os.IsNotExist(err) { - return nil, fmt.Errorf("Path of signatures file is not valid") - } - - file, err := os.Open(signatureFile) - if err != nil { - return nil, err - } - defer file.Close() - - signatureData, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - - signatures := core.NewSignatures() - - err = yaml.Unmarshal([]byte(signatureData), signatures) - if err != nil { - return nil, err - } - - severityFilter, _ := cmd.Flags().GetString("severity-filter") - if severityFilter != "" { - signatures.FilterBySeverity(severityFilter) - } - - pluginFilters, _ := cmd.Flags().GetStringSlice("plugin-filters") - if len(pluginFilters) > 0 { - signatures.FilterByNames(pluginFilters) - } - - for _, plugin := range signatures.Plugins { - if plugin.Endpoint == "" { - if len(plugin.Endpoints) > 0 { - return nil, fmt.Errorf("URI and URIs can't be set at the same time in plugin checks. Stopping execution") - } - } - for _, check := range plugin.Checks { - if check.Description == "" { - return nil, fmt.Errorf("Missing or empty description field in %s plugin checks. Stopping execution", check.Name) - } - if check.Remediation == "" { - return nil, fmt.Errorf("Missing or empty remediation field in %s plugin checks. Stopping execution", check.Name) - } - if check.Severity == "" { - return nil, fmt.Errorf("Missing severity field in %s plugin checks. Stopping execution", check.Name) - } - if !core.ValidSeverity(check.Severity) { - return nil, fmt.Errorf("Invalid severity : %s. Please use : %s", check.Severity, core.SeveritiesAsString()) - } - for _, header := range check.Headers { - if len(strings.Split(header, ":")) < 2 { - return nil, fmt.Errorf("Invalid header format : %s. Format should be KEY:VALUE", header) - } - } - } - } - - return signatures, nil -} diff --git a/core/config.go b/core/config.go deleted file mode 100644 index a9bc0ac..0000000 --- a/core/config.go +++ /dev/null @@ -1,18 +0,0 @@ -package core - -// Struct for config flags -type Config struct { - HTTP HTTPConfig - MaxSeverity string - ExportFormats []string - Urls []string - ExportFilename string - SeverityFilter string - PluginFilter []string - Threads int -} - -type HTTPConfig struct { - Insecure bool - Timeout int -} diff --git a/core/output.go b/core/output.go deleted file mode 100644 index b516719..0000000 --- a/core/output.go +++ /dev/null @@ -1,10 +0,0 @@ -package core - -// Output structure for each findings -type Output struct { - URL string `json:"url"` - Endpoint string `json:"endpoint"` - Name string `json:"checkName"` - Severity string `json:"severity"` - Remediation string `json:"remediation"` -} diff --git a/core/scan.go b/core/scan.go deleted file mode 100644 index 9469d52..0000000 --- a/core/scan.go +++ /dev/null @@ -1,149 +0,0 @@ -package core - -import ( - "context" - "fmt" - "gochopchop/internal" - "sync" - - log "github.com/sirupsen/logrus" -) - -type SafeData struct { - mux sync.Mutex - out []Output -} - -func (s *SafeData) Add(d Output) { - s.mux.Lock() - defer s.mux.Unlock() - s.out = append(s.out, d) -} - -type IFetcher interface { - Fetch(url string) (*internal.HTTPResponse, error) -} - -type IScanner interface { - Scan(urls []string) ([]Output, error) -} - -type Scanner struct { - Signatures *Signatures - Fetcher IFetcher - NoRedirectFetcher IFetcher - // Two fetchers are needed because we can't use the same http client to follow redirects - safeData *SafeData - Threads int -} - -// NewScanner returns a pointer to a initialized Scanner -func NewScanner(fetcher IFetcher, noRedirectFetcher IFetcher, signatures *Signatures, threads int) *Scanner { - safeData := &SafeData{out: make([]Output, 0)} - return &Scanner{ - Signatures: signatures, - Fetcher: fetcher, - NoRedirectFetcher: noRedirectFetcher, - safeData: safeData, - Threads: threads, - } -} - -type workerJob struct { - url string - endpoint string - plugin *Plugin -} - -func (s Scanner) Scan(ctx context.Context, urls []string) ([]Output, error) { - wg := new(sync.WaitGroup) - jobs := make(chan workerJob) - - for i := 0; i < s.Threads; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case <-ctx.Done(): - return - case job, ok := <-jobs: - if !ok { // no more jobs - return - } - resp, err := s.fetch(job.url, job.plugin.FollowRedirects) - if err != nil { - log.Error(err) - break - } - swg := new(sync.WaitGroup) - for _, check := range job.plugin.Checks { - swg.Add(1) - go func(check *Check) { - defer swg.Done() - select { - case <-ctx.Done(): - return - default: - if check.Match(resp) { - o := Output{ - URL: job.url, - Name: check.Name, - Endpoint: job.endpoint, - Severity: check.Severity, - Remediation: check.Remediation, - } - s.safeData.Add(o) - } - } - }(check) - } - swg.Wait() - } - } - }() - } - - for _, url := range urls { - for _, plugin := range s.Signatures.Plugins { - if plugin.Endpoint != "" { - plugin.Endpoints = []string{plugin.Endpoint} - } - for _, e := range plugin.Endpoints { - endpoint := e - if plugin.QueryString != "" { - endpoint = fmt.Sprintf("%s?%s", endpoint, plugin.QueryString) - } - fullURL := fmt.Sprintf("%s%s", url, endpoint) - log.Info("Testing url : ", fullURL) - - w := workerJob{url: fullURL, endpoint: endpoint, plugin: plugin} - select { - case <-ctx.Done(): - break - case jobs <- w: - } - } - } - } - - close(jobs) - wg.Wait() - - return s.safeData.out, nil -} - -func (s Scanner) fetch(url string, followRedirects bool) (*internal.HTTPResponse, error) { - var httpResponse *internal.HTTPResponse - var err error - - if !followRedirects { - httpResponse, err = s.NoRedirectFetcher.Fetch(url) - } else { - httpResponse, err = s.Fetcher.Fetch(url) - } - if err != nil { - return nil, err - } - return httpResponse, nil -} diff --git a/core/scan_test.go b/core/scan_test.go deleted file mode 100644 index 1e80e0b..0000000 --- a/core/scan_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package core_test - -import ( - "context" - "gochopchop/core" - "gochopchop/mock" - "testing" - "time" -) - -// TODO : Test fonctionnel -// TODO : integrer tests unitaires dans la CI (voir github workflows avec go test) - -func TestScan(t *testing.T) { - var tests = map[string]struct { - ctx context.Context - urls []string - output []core.Output - }{ - "no vulnerabilities found": {ctx: context.Background(), urls: []string{"http://noproblem"}, output: []core.Output{}}, - "all vulnerabilities found": {ctx: context.Background(), urls: []string{"http://problems"}, output: mock.FakeOutput}, - "context is done": {ctx: context.Background(), urls: []string{"http://noproblem"}, output: []core.Output{}}, - "fetcher problem": {ctx: context.Background(), urls: []string{"http://unknown"}, output: []core.Output{}}, - "no HTTP Response": {ctx: context.Background(), urls: []string{"http://nohttpresponse"}, output: []core.Output{}}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - - if name == "context is done" { - ctx, cancel := context.WithDeadline(tc.ctx, time.Now().Add(-7*time.Hour)) - tc.ctx = ctx - cancel() - } - - output, _ := mock.FakeScanner.Scan(tc.ctx, tc.urls) - - for _, haveOutput := range tc.output { - found := false - for _, wantOutput := range output { - if wantOutput.Name == haveOutput.Name { - found = true - break - } - } - if !found { - t.Errorf("expected: %v, got: %v", tc.output, output) - } - } - }) - } -} diff --git a/core/severity.go b/core/severity.go deleted file mode 100644 index 47be86c..0000000 --- a/core/severity.go +++ /dev/null @@ -1,40 +0,0 @@ -package core - -import "strings" - -var severities = [4]string{"High", "Medium", "Low", "Informational"} - -func ValidSeverity(severity string) bool { - for _, sv := range severities { - if severity == sv { - return true - } - } - return false -} - -func SeveritiesAsString() string { - return strings.Join(severities[:], ", ") -} - -func SeverityReached(max string, severity string) bool { - switch max { - case "High": - if severity == "High" { - return true - } - case "Medium": - if severity == "High" || severity == "Medium" { - return true - } - case "Low": - if severity == "High" || severity == "Medium" || severity == "Low" { - return true - } - case "Informational": - if severity == "High" || severity == "Medium" || severity == "Low" || severity == "Informational" { - return true - } - } - return false -} diff --git a/core/severity_test.go b/core/severity_test.go deleted file mode 100644 index fa3c60b..0000000 --- a/core/severity_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package core_test - -import ( - "testing" - "gochopchop/core" -) - -func TestValidSeverity(t *testing.T) { - var tests = map[string]struct { - severity string - want bool - }{ - "High": {severity: "High", want: true}, - "Medium": {severity: "Medium", want: true}, - "Low": {severity: "Low", want: true}, - "Informational": {severity: "Informational", want: true}, - "Bad severity": {severity: "Unknown", want: false}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - have := core.ValidSeverity(tc.severity) - if tc.want != have { - t.Errorf("expected: %v, got: %v", tc.want, have) - } - }) - } -} - -func TestSeveritiesAsString(t *testing.T) { - want := "High, Medium, Low, Informational" - have := core.SeveritiesAsString() - if have != want { - t.Errorf("expected: %v, got: %v", want, have) - } -} - -func TestSeverityReached(t *testing.T) { - var tests = map[string]struct { - max string - severity string - want bool - }{ - "HighNotReached": {max: "High", severity: "Informational", want: false}, - "HighReached": {max: "High", severity: "High", want: true}, - "MediumReached": {max: "Medium", severity: "High", want: true}, - "MediumNotReached": {max: "Medium", severity: "Low", want: false}, - "LowReached": {max: "Low", severity: "High", want: true}, - "LowNotReached": {max: "Low", severity: "Informational", want: false}, - "InformationalReached": {max: "Informational", severity: "Informational", want: true}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - have := core.SeverityReached(tc.max, tc.severity) - if tc.want != have { - t.Errorf("want: %v, have: %v", tc.want, have) - } - }) - } -} diff --git a/core/signatures.go b/core/signatures.go deleted file mode 100644 index b8419d1..0000000 --- a/core/signatures.go +++ /dev/null @@ -1,253 +0,0 @@ -package core - -import ( - "gochopchop/internal" - "strings" -) - -// Signature struct to load the plugins/rules from the YAML file -type Signatures struct { - Plugins []*Plugin `yaml:"plugins"` -} - -type Plugin struct { - Endpoints []string `yaml:"endpoints"` - Endpoint string `yaml:"endpoint"` - QueryString string `yaml:"query_string"` - Checks []*Check `yaml:"checks"` - FollowRedirects bool `yaml:"follow_redirects"` -} - -// Check Signature -type Check struct { - MustMatchOne []string `yaml:"match"` - MustMatchAll []string `yaml:"all_match"` - MustNotMatch []string `yaml:"no_match"` - StatusCode *int32 `yaml:"status_code"` - Name string `yaml:"name"` - Remediation string `yaml:"remediation"` - Severity string `yaml:"severity"` - Description string `yaml:"description"` - Headers []string `yaml:"headers"` - NoHeaders []string `yaml:"no_headers"` -} - -// NewSignatures returns a new initialized Signatures -func NewSignatures() *Signatures { - return &Signatures{} -} - -func (s *Signatures) FilterBySeverity(severity string) { - filteredPlugins := s.Plugins[:0] - for _, plugin := range s.Plugins { - filteredChecks := plugin.Checks[:0] - for _, check := range plugin.Checks { - if check.Severity == severity { - filteredChecks = append(filteredChecks, check) - } - } - if len(filteredChecks) > 0 { - plugin.Checks = filteredChecks - filteredPlugins = append(filteredPlugins, plugin) - } - } - s.Plugins = filteredPlugins -} - -func (s *Signatures) FilterByNames(names []string) { - filteredPlugins := s.Plugins[:0] - for _, plugin := range s.Plugins { - filteredChecks := plugin.Checks[:0] - for _, check := range plugin.Checks { - for _, name := range names { - if strings.Contains(strings.ToLower(check.Name), strings.ToLower(name)) { - filteredChecks = append(filteredChecks, check) - break - } - } - } - if len(filteredChecks) > 0 { - plugin.Checks = filteredChecks - filteredPlugins = append(filteredPlugins, plugin) - } - } - s.Plugins = filteredPlugins -} - -//Match analyses the HTTP Request -// a match means that one of the criteria has been met -func (check *Check) Match(resp *internal.HTTPResponse) bool { - // status code must match - if check.StatusCode != nil { - if int32(resp.StatusCode) != *check.StatusCode { - return false - } - } - - // all element must be found - for _, match := range check.MustMatchAll { - if !strings.Contains(resp.Body, match) { - return false - } - } - - // one element must be found - if len(check.MustMatchOne) > 0 { - found := false - for _, match := range check.MustMatchOne { - if strings.Contains(resp.Body, match) { - found = true - } - } - if !found { - return false - } - } - - // no element should match - if len(check.MustNotMatch) > 0 { - for _, match := range check.MustNotMatch { - if strings.Contains(resp.Body, match) { - return false - } - } - } - - // must contain all these headers - for _, header := range check.Headers { - pHeaders := strings.Split(header, ":") - pHeadersKey := pHeaders[0] - pHeadersValue := pHeaders[1] - if respHeaderValues, kFound := resp.Header[pHeadersKey]; kFound { - vFound := false - for _, respHeaderValue := range respHeaderValues { - if strings.Contains(respHeaderValue, pHeadersValue) { - vFound = true - break - } - } - if !vFound { - return false - } - } else { - return false - } - } - - // must not contain these headers - for _, header := range check.NoHeaders { - pNoHeaders := strings.Split(header, ":") - pNoHeadersKey := pNoHeaders[0] - if respHeaderValues, kFound := resp.Header[pNoHeadersKey]; kFound { - if len(pNoHeaders) > 1 { - pHeadersValue := pNoHeaders[1] - vFound := false - for _, respHeaderValue := range respHeaderValues { - if strings.Contains(respHeaderValue, pHeadersValue) { - vFound = true - break - } - } - if vFound { - return false - } - } - } - } - return true -} - -func (self *Signatures) Equals(signatures *Signatures) bool { - if len(self.Plugins) != len(signatures.Plugins) { - return false - } - for _, plugin := range self.Plugins { - found := false - for _, oplugin := range signatures.Plugins { - if plugin.Equals(oplugin) { - found = true - break - } - } - if !found { - return false - } - } - return true -} - -func (self *Plugin) Equals(plugin *Plugin) bool { - if !SliceStringEqual(self.Endpoints, plugin.Endpoints) { - return false - } - if self.Endpoint != plugin.Endpoint { - return false - } - if self.QueryString != plugin.QueryString { - return false - } - if self.FollowRedirects != plugin.FollowRedirects { - return false - } - for _, check := range self.Checks { - found := false - for _, pcheck := range plugin.Checks { - if check.Equals(pcheck) { - found = true - break - } - } - if !found { - return false - } - } - return true -} - -func (self *Check) Equals(check *Check) bool { - if !SliceStringEqual(self.MustMatchOne, check.MustMatchOne) { - return false - } - if !SliceStringEqual(self.MustMatchAll, check.MustMatchAll) { - return false - } - if !SliceStringEqual(self.MustNotMatch, check.MustNotMatch) { - return false - } - if self.StatusCode != nil && check.StatusCode != nil { - if *self.StatusCode != *check.StatusCode { - return false - } - } - if self.Name != check.Name { - return false - } - if self.Remediation != check.Remediation { - return false - } - if self.Severity != check.Severity { - return false - } - if self.Description != check.Description { - return false - } - if !SliceStringEqual(self.Headers, check.Headers) { - return false - } - if !SliceStringEqual(self.NoHeaders, check.NoHeaders) { - return false - } - return true -} - -func SliceStringEqual(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} diff --git a/core/signatures_test.go b/core/signatures_test.go deleted file mode 100644 index ea24113..0000000 --- a/core/signatures_test.go +++ /dev/null @@ -1,285 +0,0 @@ -package core_test - -import ( - "gochopchop/core" - "gochopchop/mock" - "testing" -) - -func TestFilterBySeverity(t *testing.T) { - var tests = map[string]struct { - have *core.Signatures - want *core.Signatures - severity string - }{ - "Filter nothing": { - have: &core.Signatures{Plugins: []*core.Plugin{mock.FakePlugin}}, - want: &core.Signatures{Plugins: []*core.Plugin{mock.FakePlugin}}, - severity: "Medium", - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - tc.have.FilterBySeverity(tc.severity) - if !tc.want.Equals(tc.have) { - t.Errorf("expected: %v, got: %v", tc.want, tc.have) - } - }) - } -} - -func TestFilterByNames(t *testing.T) { - var tests = map[string]struct { - have *core.Signatures - want *core.Signatures - names []string - }{ - "Filter nothing": { - have: &core.Signatures{Plugins: []*core.Plugin{mock.FakeQueryPlugin}}, - want: &core.Signatures{Plugins: []*core.Plugin{mock.FakeQueryPlugin}}, - names: []string{mock.FakeCheckStatusCode200.Name}, - }, - "Filter one element": { - have: &core.Signatures{Plugins: []*core.Plugin{mock.FakeQueryPlugin}}, - want: &core.Signatures{}, - names: []string{"check's name that is not in the signatures"}, - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - tc.have.FilterByNames(tc.names) - if !tc.have.Equals(tc.want) { - t.Errorf("expected: %v, got: %v", tc.want, tc.have) - } - }) - } -} - -func TestPluginEquals(t *testing.T) { - var tests = map[string]struct { - plugin1 *core.Plugin - plugin2 *core.Plugin - want bool - }{ - "Different Endpoints": { - plugin1: &core.Plugin{ - Endpoint: "/endpoint1", - }, - plugin2: &core.Plugin{ - Endpoint: "/endpoint2", - }, - want: false, - }, - "Different Query String": { - plugin1: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test1", - }, - plugin2: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test2", - }, - want: false, - }, - "Different Follow Redirects Bool": { - plugin1: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test1", - FollowRedirects: true, - }, - plugin2: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test1", - FollowRedirects: false, - }, - want: false, - }, - "Equals Checks": { - plugin1: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test1", - FollowRedirects: true, - Checks: []*core.Check{ - mock.FakeCheckStatusCode200, - }, - }, - plugin2: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test1", - FollowRedirects: true, - Checks: []*core.Check{ - mock.FakeCheckStatusCode200, - }, - }, - want: true, - }, - "Not Equals Checks": { - plugin1: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test1", - FollowRedirects: true, - Checks: []*core.Check{ - mock.FakeCheckStatusCode200, - mock.FakeCheckMatchAll, - }, - }, - plugin2: &core.Plugin{ - Endpoint: "/endpoint1", - QueryString: "query=test1", - FollowRedirects: true, - Checks: []*core.Check{ - mock.FakeCheckStatusCode500, - mock.FakeCheckMatchAll, - }, - }, - want: false, - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - have := tc.plugin1.Equals(tc.plugin2) - if have != tc.want { - t.Errorf("expected: %v, got: %v", tc.want, have) - } - }) - } -} - -func TestSignaturesEquals(t *testing.T) { - var tests = map[string]struct { - signatures1 *core.Signatures - signatures2 *core.Signatures - want bool - }{ - "Different Length": { - signatures1: &core.Signatures{Plugins: []*core.Plugin{ - mock.FakePlugin, - mock.FakeQueryPlugin, - mock.FakeFollowRedirectPlugin, - }}, - signatures2: core.NewSignatures(), - want: false, - }, - - "Not the same plugin content": { - signatures1: &core.Signatures{Plugins: []*core.Plugin{ - mock.FakePlugin2, - mock.FakeQueryPlugin, - mock.FakeFollowRedirectPlugin, - }}, - signatures2: &core.Signatures{Plugins: []*core.Plugin{ - mock.FakePlugin, - mock.FakeQueryPlugin, - mock.FakeFollowRedirectPlugin, - }}, - want: false, - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - have := tc.signatures1.Equals(tc.signatures2) - if have != tc.want { - t.Errorf("expected: %v, got: %v", tc.want, have) - } - }) - } -} -func TestCheckEquals(t *testing.T) { - var tests = map[string]struct { - check1 *core.Check - check2 *core.Check - want bool - }{ - "MustMatchOne not Equals": { - check1: &core.Check{MustMatchOne: []string{"MATCHONE", "MATCHTWO"}}, - check2: &core.Check{MustMatchOne: []string{"MATCHFOUR", "MATCHTHREE"}}, - want: false, - }, - "MustMatchAll not Equals": { - check1: &core.Check{MustMatchAll: []string{"MATCHONE", "MATCHTWO"}}, - check2: &core.Check{MustMatchAll: []string{"MATCHONE", "MATCHTHREE"}}, - want: false, - }, - "MustNotMatch Equals": { - check1: &core.Check{MustNotMatch: []string{"MATCHONE", "MATCHTWO"}}, - check2: &core.Check{MustNotMatch: []string{"MATCHONE", "MATCHTHREE"}}, - want: false, - }, - "Name not Equals": { - check1: &core.Check{Name: "Name1"}, - check2: &core.Check{Name: "Name2"}, - want: false, - }, - "Remediation not Equals": { - check1: &core.Check{Remediation: "ಠ_ಠ"}, - check2: &core.Check{Remediation: "(°_o)"}, - want: false, - }, - "Severity not Equals": { - check1: &core.Check{Severity: "High"}, - check2: &core.Check{Severity: "Medium"}, - want: false, - }, - "Description not Equals": { - check1: &core.Check{Description: "ಠ_ಠ"}, - check2: &core.Check{Description: "(°_o)"}, - want: false, - }, - "Headers not Equals": { - check1: &core.Check{Headers: []string{"Header:OK"}}, - check2: &core.Check{Headers: []string{"Header:notOK"}}, - want: false, - }, - "NoHeaders not Equals": { - check1: &core.Check{NoHeaders: []string{"NoHeader:OK"}}, - check2: &core.Check{NoHeaders: []string{"NoHeader:notOK"}}, - want: false, - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - have := tc.check1.Equals(tc.check2) - if have != tc.want { - t.Errorf("expected: %v, got: %v", tc.want, have) - } - }) - } -} - -func TestSliceStringEqual(t *testing.T) { - var tests = map[string]struct { - slice1 []string - slice2 []string - want bool - }{ - "Same slices": { - slice1: []string{"a", "b"}, - slice2: []string{"a", "b"}, - want: true, - }, - "Different slices with same length": { - slice1: []string{"a", "b"}, - slice2: []string{"x", "b"}, - want: false, - }, - "Different slices with different length": { - slice1: []string{"a", "b"}, - slice2: []string{"a", "b", "c"}, - want: false, - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - have := core.SliceStringEqual(tc.slice1, tc.slice2) - if have != tc.want { - t.Errorf("expected: %v, got: %v", tc.want, have) - } - }) - } -} diff --git a/docs/img/demo.gif b/docs/img/demo.gif index cacbf4d..d9db094 100644 Binary files a/docs/img/demo.gif and b/docs/img/demo.gif differ diff --git a/go.mod b/go.mod index ec9086e..b1c00c3 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,14 @@ -module gochopchop +module github.com/michelin/gochopchop -go 1.15 +go 1.16 require ( - github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect - github.com/go-openapi/errors v0.19.8 // indirect - github.com/go-openapi/strfmt v0.19.8 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/go-openapi/strfmt v0.20.1 // indirect + github.com/google/go-cmp v0.5.2 github.com/jedib0t/go-pretty v4.3.0+incompatible - github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mitchellh/mapstructure v1.3.3 // indirect - github.com/sirupsen/logrus v1.7.0 - github.com/spf13/afero v1.1.2 - github.com/spf13/cobra v1.1.1 - go.mongodb.org/mongo-driver v1.4.3 // indirect - golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65 // indirect - gopkg.in/yaml.v2 v2.3.0 + github.com/mattn/go-runewidth v0.0.12 // indirect + github.com/sirupsen/logrus v1.8.1 + github.com/urfave/cli/v2 v2.3.0 + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 9f59263..adc286f 100644 --- a/go.sum +++ b/go.sum @@ -1,60 +1,18 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/errors v0.19.8 h1:doM+tQdZbUm9gydV9yR+iQNmztbjj7I3sW4sIcAwIzc= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/strfmt v0.19.8 h1:9wAdSoImc5UCnUj79GhcjkJXxuU/nEwlbe6SKe9ZdRs= -github.com/go-openapi/strfmt v0.19.8/go.mod h1:qBBipho+3EoIqn6YDI+4RnQEtj6jT/IdKm+PAlXxSUc= +github.com/go-openapi/strfmt v0.20.1 h1:1VgxvehFne1mbChGeCmZ5pc0LxUf6yaACVSIYAR91Xc= +github.com/go-openapi/strfmt v0.20.1/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -82,326 +40,114 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.mongodb.org/mongo-driver v1.4.2/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.4.3 h1:moga+uhicpVshTyaqY9L23E6QqwcHRUv1sqyOsoyOO8= -go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= +go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65 h1:Qo9oJ566/Sq7N4hrGftVXs8GI2CXBCuOd4S2wHE/e0M= -golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/config.go b/internal/config.go new file mode 100644 index 0000000..865ad52 --- /dev/null +++ b/internal/config.go @@ -0,0 +1,196 @@ +package internal + +import ( + "bufio" + "errors" + "fmt" + "io" + "net/url" + "strconv" + "time" +) + +// Config wraps all the parameters to configure +// a scan. +type Config struct { + HTTP HTTPConfig + MaxSeverity Severity + SeverityFilter Severity + ExportFormats []string + PluginFilter []string + Urls []string + ExportFilename string + Goroutines int +} + +// HTTPConfig wraps HTTP configurations parameters. +type HTTPConfig struct { + Insecure bool + Timeout int +} + +// ErrNoURL is an error meaning no url is provided. +var ErrNoURL = errors.New("no urls provided neither with the flag --url-list, -u neither in args") + +// ErrBothURLAndURLList is an error meaning both args +// and url-file are provided. +var ErrBothURLAndURLList = errors.New("urls provided either with the flag --url-list, -u either in args") + +// ErrInvalidURLs is an error meaning url(s) is(are) +// invalid. +type ErrInvalidURLs struct { + URLs []string +} + +func (e ErrInvalidURLs) Error() string { + s := "invalid URLs: " + l := len(e.URLs) + for i := 0; i < l; i++ { + s += e.URLs[i] + if i != l-1 { + s += ", " + } + } + return s +} + +// ErrInvalidExport is an error meaning export(s) is(are) +// not matching allowed exports. +type ErrInvalidExport struct { + Exports []string +} + +func (e ErrInvalidExport) Error() string { + s := "invalid exports: " + l := len(e.Exports) + for i := 0; i < l; i++ { + s += e.Exports[i] + if i != l-1 { + s += ", " + } + } + return s +} + +// ErrFailedOperationOnField is an error meaning +// a field has failed to pass an operation on a value. +type ErrFailedOperationOnField struct { + Field string + Operation string + Value int +} + +func (e ErrFailedOperationOnField) Error() string { + return e.Field + " failed to be " + e.Operation + " (specified " + strconv.Itoa(e.Value) + ")" +} + +// BuildConfig builds the core.Config from provided values. +// Those are supposed to come from the "scan" command flags. +func BuildConfig(insecure bool, export, pluginFilters []string, exportFilename, maxSeverity, severityFilter string, urlFile io.Reader, threads, timeout int, args []string) (*Config, error) { + nArg := len(args) + + // Check insecure => always fine + + // Check export + var invalidExport []string + for _, e := range export { + if _, ok := exportersMap[e]; !ok { + invalidExport = append(invalidExport, e) + } + } + if len(invalidExport) != 0 { + return nil, &ErrInvalidExport{invalidExport} + } + + // Check export-filename + if exportFilename == "" { + now := time.Now().Format("2006-01-02_15-04-05") + exportFilename = fmt.Sprintf("gochopchop_%s", now) + } + + // Check severities + sevFilter, err := StringToSeverity(severityFilter) + if err != nil { + return nil, err + } + maxSev, err := StringToSeverity(maxSeverity) + if err != nil { + return nil, err + } + + // Check url conditions + var urls []string + var invalidUrls []string + if urlFile == nil { + if nArg == 0 { + // There are no args (urls) to chopchop + return nil, ErrNoURL + } + + // Check URLs validity + for i := 0; i < nArg; i++ { + arg := args[i] + if !isValidUrl(arg) { + invalidUrls = append(invalidUrls, arg) + continue + } + urls = append(urls, arg) + } + } else { + // Check there are not args (urls) and an url-file to chopchop + if nArg != 0 { + return nil, ErrBothURLAndURLList + } + + // Read content and add if is a valid url + scanner := bufio.NewScanner(urlFile) + for scanner.Scan() { + url := scanner.Text() + if !isValidUrl(url) { + invalidUrls = append(invalidUrls, url) + continue + } + urls = append(urls, url) + } + + // Ensure there were no issues while scanning + if err := scanner.Err(); err != nil { + return nil, err + } + } + if len(invalidUrls) != 0 { + return nil, &ErrInvalidURLs{invalidUrls} + } + + // Check threads + if threads <= 0 { + return nil, &ErrFailedOperationOnField{"threads", "<=0", threads} + } + + // Check timeout + if timeout < 0 { + return nil, &ErrFailedOperationOnField{"timeout", "<0", timeout} + } + + // Build config + config := &Config{ + HTTP: HTTPConfig{ + Insecure: insecure, + Timeout: timeout, + }, + MaxSeverity: maxSev, + SeverityFilter: sevFilter, + ExportFormats: export, + PluginFilter: pluginFilters, + Urls: urls, + ExportFilename: exportFilename, + Goroutines: threads, + } + + return config, nil +} + +func isValidUrl(urlStr string) bool { + u, err := url.Parse(urlStr) + return err == nil && u.Scheme != "" && u.Host != "" +} diff --git a/internal/config_test.go b/internal/config_test.go new file mode 100644 index 0000000..2fe69ec --- /dev/null +++ b/internal/config_test.go @@ -0,0 +1,174 @@ +package internal_test + +import ( + "io" + "reflect" + "testing" + + "github.com/michelin/gochopchop/internal" +) + +type FakeReadCloser struct { + data []byte + readIndex int64 +} + +func NewFakeReadCloser(toRead string) *FakeReadCloser { + return &FakeReadCloser{data: []byte(toRead)} +} + +func (f *FakeReadCloser) Read(p []byte) (n int, err error) { + if f.readIndex >= int64(len(f.data)) { + err = io.EOF + return + } + + n = copy(p, f.data[f.readIndex:]) + f.readIndex += int64(n) + return +} + +func (f *FakeReadCloser) Close() error { + return nil +} + +var _ = (io.ReadCloser)(&FakeReadCloser{}) + +func TestBuildConfig(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Insecure bool + Export []string + PluginFilters []string + ExportFilename string + MaxSeverity string + SeverityFilter string + URLFile io.Reader + Threads int + Timeout int + Args []string + ExpectedConfig *internal.Config + ExpectedErr error + }{ + "inexisting-exporter": { + Export: []string{"inexisting-exporter", "other-inexisting-exporter"}, + ExpectedConfig: nil, + ExpectedErr: &internal.ErrInvalidExport{[]string{"inexisting-exporter", "other-inexisting-exporter"}}, + }, + "invalid-severityfilter": { + SeverityFilter: "invalid-severity", + ExpectedConfig: nil, + ExpectedErr: &internal.ErrInvalidSeverity{"invalid-severity"}, + }, + "invalid-maxseverity": { + SeverityFilter: "Low", + MaxSeverity: "invalid-severity", + ExpectedConfig: nil, + ExpectedErr: &internal.ErrInvalidSeverity{"invalid-severity"}, + }, + "nil-urlfile-no-args": { + SeverityFilter: "Low", + MaxSeverity: "Low", + URLFile: nil, + Args: []string{}, + ExpectedConfig: nil, + ExpectedErr: internal.ErrNoURL, + }, + "nil-urlfile-args-invalid-urls": { + SeverityFilter: "Low", + MaxSeverity: "Low", + URLFile: nil, + Args: []string{"https://www.michelin.com/", "gochopchop", "ChopChop"}, + ExpectedConfig: nil, + ExpectedErr: &internal.ErrInvalidURLs{[]string{"gochopchop", "ChopChop"}}, + }, + "urlfile-and-args": { + SeverityFilter: "Low", + MaxSeverity: "Low", + URLFile: NewFakeReadCloser(""), + Args: []string{""}, + ExpectedConfig: nil, + ExpectedErr: internal.ErrBothURLAndURLList, + }, + "urlfile-no-args-invalid-urls": { + SeverityFilter: "Low", + MaxSeverity: "Low", + URLFile: NewFakeReadCloser("https://www.michelin.com/\ngochopchop\nChopChop\n"), + Args: []string{}, + ExpectedConfig: nil, + ExpectedErr: &internal.ErrInvalidURLs{[]string{"gochopchop", "ChopChop"}}, + }, + "url-file-args-invalid-urls-scanner-err": { + SeverityFilter: "Low", + MaxSeverity: "Low", + URLFile: &FailingReadCloser{}, + Args: []string{}, + ExpectedConfig: nil, + ExpectedErr: errFake, + }, + "negative-threads": { + SeverityFilter: "Low", + MaxSeverity: "Low", + Threads: -1, + Args: []string{"https://www.michelin.com/"}, + ExpectedConfig: nil, + ExpectedErr: &internal.ErrFailedOperationOnField{"threads", "<=0", -1}, + }, + "zero-threads": { + SeverityFilter: "Low", + MaxSeverity: "Low", + Threads: 0, + Args: []string{"https://www.michelin.com/"}, + ExpectedConfig: nil, + ExpectedErr: &internal.ErrFailedOperationOnField{"threads", "<=0", 0}, + }, + "negative-timeout": { + SeverityFilter: "Low", + MaxSeverity: "Low", + Threads: 1, + Timeout: -1, + Args: []string{"https://www.michelin.com/"}, + ExpectedConfig: nil, + ExpectedErr: &internal.ErrFailedOperationOnField{"timeout", "<0", -1}, + }, + "valid-config": { + Insecure: false, + Export: []string{"stdout", "csv", "json"}, + PluginFilters: []string{}, + ExportFilename: "results", + MaxSeverity: "High", + SeverityFilter: "Informational", + URLFile: nil, + Threads: 1, + Timeout: 0, + Args: []string{"https://www.michelin.com/"}, + ExpectedConfig: &internal.Config{ + HTTP: internal.HTTPConfig{ + Insecure: false, + Timeout: 0, + }, + MaxSeverity: internal.High, + SeverityFilter: internal.Informational, + ExportFormats: []string{"stdout", "csv", "json"}, + PluginFilter: []string{}, + Urls: []string{"https://www.michelin.com/"}, + ExportFilename: "results", + Goroutines: 1, + }, + ExpectedErr: nil, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + config, err := internal.BuildConfig(tt.Insecure, tt.Export, tt.PluginFilters, tt.ExportFilename, tt.MaxSeverity, tt.SeverityFilter, tt.URLFile, tt.Threads, tt.Timeout, tt.Args) + + if !reflect.DeepEqual(config, tt.ExpectedConfig) { + t.Errorf("Failed to get expected Config: got \"%v\" intead of \"%v\".", config, tt.ExpectedConfig) + } + + checkErr(err, tt.ExpectedErr, t) + }) + } +} diff --git a/internal/errors.go b/internal/errors.go new file mode 100644 index 0000000..e94950d --- /dev/null +++ b/internal/errors.go @@ -0,0 +1,10 @@ +package internal + +// ErrNilParameter is an error meaning a parameter is nil. +type ErrNilParameter struct { + Name string +} + +func (e ErrNilParameter) Error() string { + return "parameter " + e.Name + " is nil" +} diff --git a/internal/errors_test.go b/internal/errors_test.go new file mode 100644 index 0000000..960c64f --- /dev/null +++ b/internal/errors_test.go @@ -0,0 +1,121 @@ +package internal_test + +import ( + "errors" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/michelin/gochopchop/internal" + "gopkg.in/yaml.v2" +) + +var errStrTypeOf = reflect.TypeOf(errors.New("")) + +func checkErr(err, expErr error, t *testing.T) { + // Check err type + typeErr := reflect.TypeOf(err) + typeExpErr := reflect.TypeOf(expErr) + if typeErr != typeExpErr { + t.Fatalf("Failed to get expected error type: got \"%s\" instead of \"%s\".", typeErr, typeExpErr) + } + + // Check Error content is not empty + if err != nil && err.Error() == "" { + t.Error("Error should not have an empty content.") + } + + // Check err fields. Takes the asumption that they are + // of the same type, thanks to the previous check. + if typeErr == errStrTypeOf { + return // Skip following checks for errors generated by errors.New + } + + switch err.(type) { + case *internal.ErrNilParameter: + castedErr := err.(*internal.ErrNilParameter) + castedExpErr := expErr.(*internal.ErrNilParameter) + + if castedErr.Name != castedExpErr.Name { + t.Error("Failed to get expected ErrNilParameter.Name: got \"" + castedErr.Name + "\" instead of \"" + castedExpErr.Name + "\".") + } + + case *internal.ErrUnsupportedSeverity: + castedErr := err.(*internal.ErrUnsupportedSeverity) + castedExpErr := expErr.(*internal.ErrUnsupportedSeverity) + + if castedErr.Severity != castedExpErr.Severity { + t.Errorf("Failed to get expected ErrUnsupportedSeverity.Severity: got \"%d\" instead of \"%d\".", castedErr.Severity, castedExpErr.Severity) + } + + case *internal.ErrInvalidSeverity: + castedErr := err.(*internal.ErrInvalidSeverity) + castedExpErr := expErr.(*internal.ErrInvalidSeverity) + + if castedErr.Severity != castedExpErr.Severity { + t.Error("Failed to get expected ErrInvalidSeverity.Severity: got \"" + castedErr.Severity + "\" intead of \"" + castedExpErr.Severity + "\".") + } + + case *internal.ErrInvalidHeaderFormat: + castedErr := err.(*internal.ErrInvalidHeaderFormat) + castedExpErr := expErr.(*internal.ErrInvalidHeaderFormat) + + if castedErr.Header != castedExpErr.Header { + t.Error("Failed to get expected ErrInvalidSeverity.Header: got \"" + castedErr.Header + "\" intead of \"" + castedExpErr.Header + "\".") + } + + case *internal.ErrInvalidExport: + castedErr := err.(*internal.ErrInvalidExport) + castedExpErr := expErr.(*internal.ErrInvalidExport) + + if !cmp.Equal(castedErr.Exports, castedExpErr.Exports) { + t.Errorf("Failed to get expected ErrInvalidExport.Exports: got \"%v\" instead of \"%v\".", castedErr.Exports, castedExpErr.Exports) + } + + case *internal.ErrInvalidURLs: + castedErr := err.(*internal.ErrInvalidURLs) + castedExpErr := expErr.(*internal.ErrInvalidURLs) + + if !cmp.Equal(castedErr.URLs, castedExpErr.URLs) { + t.Errorf("Failed to get expected ErrInvalidURLs.URLs: got \"%v\" instead of \"%v\".", castedErr.URLs, castedExpErr.URLs) + } + + case *internal.ErrFailedOperationOnField: + castedErr := err.(*internal.ErrFailedOperationOnField) + castedExpErr := expErr.(*internal.ErrFailedOperationOnField) + + if castedErr.Field != castedExpErr.Field { + t.Error("Failed to get expected ErrFailedOperationOnField.Field: got \"" + castedErr.Field + "\" instead of \"" + castedExpErr.Field + "\".") + } + if castedErr.Operation != castedExpErr.Operation { + t.Error("Failed to get expected ErrFailedOperationOnField.Operation: got \"" + castedErr.Operation + "\" instead of \"" + castedExpErr.Operation + "\".") + } + if castedErr.Value != castedExpErr.Value { + t.Errorf("Failed to get expected ErrFailedOperationOnField.Value: got \"%d\" instead of \"%d\".", castedErr.Value, castedExpErr.Value) + } + + case *internal.ErrCheckInvalidField: + castedErr := err.(*internal.ErrCheckInvalidField) + castedExpErr := expErr.(*internal.ErrCheckInvalidField) + + if castedErr.Check != castedExpErr.Check { + t.Error("Failed to get expected ErrFailedOperationOnField.Field: got \"" + castedErr.Check + "\" instead of \"" + castedExpErr.Check + "\".") + } + if castedErr.Field != castedExpErr.Field { + t.Error("Failed to get expected ErrFailedOperationOnField.Field: got \"" + castedErr.Field + "\" instead of \"" + castedExpErr.Field + "\".") + } + + case *internal.ErrNilFetcher: + castedErr := err.(*internal.ErrNilFetcher) + castedExpErr := expErr.(*internal.ErrNilFetcher) + + if castedErr.FetcherName != castedExpErr.FetcherName { + t.Error("Failed to get expected ErrNilFetcher.Name: got \"" + castedErr.FetcherName + "\" instead of \"" + castedExpErr.FetcherName + "\".") + } + + case nil, *yaml.TypeError: + // It's fine there + default: + t.Logf("\033[31mcheckErr Unsupported type: %s\033[0m\n", typeErr) + } +} diff --git a/internal/export.go b/internal/export.go new file mode 100644 index 0000000..556e84a --- /dev/null +++ b/internal/export.go @@ -0,0 +1,244 @@ +package internal + +import ( + "encoding/json" + "errors" + "io" + "os" + + "github.com/jedib0t/go-pretty/table" +) + +// ExporterFunc is a func type exporting the results +// to a writer. +type ExporterFunc func([]Result, io.WriteCloser) error + +var exportersMap = map[string]struct { + ExporterFunc ExporterFunc + WriterProvider func(filename string) (io.WriteCloser, error) +}{ + "csv": { + ExporterFunc: ExportCSV, + WriterProvider: func(filename string) (io.WriteCloser, error) { + return os.OpenFile(filename+".csv", os.O_CREATE|os.O_WRONLY, 0644) + }, + }, + "json": { + ExporterFunc: ExportJSON, + WriterProvider: func(filename string) (io.WriteCloser, error) { + return os.OpenFile(filename+".json", os.O_CREATE|os.O_WRONLY, 0644) + }, + }, + "stdout": { + ExporterFunc: ExportTableColor, + WriterProvider: func(filename string) (io.WriteCloser, error) { + return os.Stdout, nil + }, + }, + "stdout-no-color": { + ExporterFunc: ExportTableNoColor, + WriterProvider: func(filename string) (io.WriteCloser, error) { + return os.Stdout, nil + }, + }, +} + +func ExportersList() string { + s := "" + l := len(exportersMap) + i := 0 + for exp := range exportersMap { + s += exp + if i < l-1 { + s += ", " + } + i++ + } + return s +} + +// ErrEmptyResults is an error meaning the results are empty. +var ErrEmptyResults = errors.New("no result found") + +// ErrMaxSeverityReached is an error meaning a result severity +// has been reached. +type ErrMaxSeverityReached struct { + Max, Sev Severity +} + +func (e ErrMaxSeverityReached) Error() string { + maxStr, _ := e.Max.String() + sevStr, _ := e.Sev.String() + return "max severity (" + maxStr + ") reached (" + sevStr + ")" +} + +func CheckSeverities(results []Result, max Severity) error { + _, err := max.String() + if err != nil { + return err + } + + // Check results severities are valid + for _, res := range results { + sevRes, err := StringToSeverity(res.Severity) + if err != nil { + return err + } + if sevRes < max { + return &ErrMaxSeverityReached{Max: max, Sev: sevRes} + } + } + return nil +} + +// ErrUnsupportedExporter is an error meaning an exporter in +// a Config is not supported. +type ErrUnsupportedExporter struct { + Exporter string +} + +func (e ErrUnsupportedExporter) Error() string { + return "unsupported exporter: " + e.Exporter +} + +// ExportResults exports the results given a config, to a filename +// if the exporter needs it. +func ExportResults(results []Result, config *Config, filename string) error { + // Check parameters + if config == nil { + return &ErrNilParameter{"config"} + } + if len(results) == 0 { + return ErrEmptyResults + } + + // Check severities + err := CheckSeverities(results, config.MaxSeverity) + if err != nil { + return err + } + + // Export results + exported := make(map[string]struct{}) + for _, format := range config.ExportFormats { + if _, ok := exported[format]; !ok { + exported[format] = struct{}{} + + d := exportersMap[format] + f, err := d.WriterProvider(filename) + if err != nil { + return err + } + defer f.Close() + + err = d.ExporterFunc(results, f) + if err != nil { + return err + } + } + } + + return nil +} + +func ExportJSON(results []Result, w io.WriteCloser) error { + // Marshal results in JSON + jsonbytes, err := json.Marshal(results) + if err != nil { + return err + } + + // Write JSON content in file + if _, err := w.Write(jsonbytes); err != nil { + return err + } + + return nil +} + +func ExportCSV(results []Result, w io.WriteCloser) error { + // Write headers + _, err := w.Write([]byte("url,endpoint,severity,checkName,remediation\n")) + if err != nil { + return err + } + + // Write content + for _, result := range results { + entry := result.URL + "," + result.Endpoint + "," + result.Severity + "," + result.Name + "," + result.Remediation + "\n" + _, err := w.Write([]byte(entry)) + if err != nil { + return err + } + } + + return nil +} + +const ( + colorReset = "\033[0m" + colorRed = "\033[31m" + colorGreen = "\033[32m" + colorYellow = "\033[33m" + colorCyan = "\033[36m" +) + +func ExportTableColor(results []Result, w io.WriteCloser) error { + return exportTable(results, w, true) +} + +func ExportTableNoColor(results []Result, w io.WriteCloser) error { + return exportTable(results, w, false) +} + +func exportTable(results []Result, w io.WriteCloser, color bool) error { + t := table.NewWriter() + t.SetOutputMirror(w) + t.AppendHeader(table.Row{"URL", "Endpoint", "Severity", "Plugin", "Remediation"}) + for _, result := range results { + // Convert and check severity + sev, err := StringToSeverity(result.Severity) + if err != nil { + return err + } + + // Build log severity + var severity string + switch sev { + case High: + severity = colorify(colorRed, "High", color) + case Medium: + severity = colorify(colorYellow, "Medium", color) + case Low: + severity = colorify(colorGreen, "Low", color) + case Informational: + severity = colorify(colorCyan, "Informational", color) + } + + // Append the content row + t.AppendRow([]interface{}{ + result.URL, + result.Endpoint, + severity, + result.Name, + result.Remediation, + }) + } + t.SortBy([]table.SortBy{ + {Name: "Severity", Mode: table.Asc}, + }) + t.Render() + + return nil +} + +func colorify(color string, str string, colorify bool) (s string) { + if colorify { + s += color + } + s += str + if colorify { + s += colorReset + } + return +} diff --git a/internal/export/export.go b/internal/export/export.go deleted file mode 100644 index a84abc7..0000000 --- a/internal/export/export.go +++ /dev/null @@ -1,77 +0,0 @@ -package export - -import ( - "encoding/json" - "fmt" - "gochopchop/core" - "os" - - log "github.com/sirupsen/logrus" -) - -type IFile interface { - WriteString(input string) (n int, err error) -} - -// ExportCSV exports the output in a CSV file -func ExportCSV(filename string, out []core.Output) error { - exportFilename := fmt.Sprintf("%s.csv", filename) - - f, err := os.OpenFile(exportFilename, os.O_CREATE|os.O_WRONLY, 0755) - defer f.Close() - if err != nil { - return err - } - - err = exportCSV(f, out) - if err != nil { - return err - } - log.Info("Results were exported as csv in: ", exportFilename) - return nil -} - -func exportCSV(file IFile, out []core.Output) error { - _, err := file.WriteString("url,endpoint,severity,checkName,remediation\n") - if err != nil { - return err - } - for _, output := range out { - line := fmt.Sprintf("%s,%s,%s,%s,%s\n", output.URL, output.Endpoint, output.Severity, output.Name, output.Remediation) - _, err := file.WriteString(line) - if err != nil { - return err - } - } - - return nil -} - -// ExportJSON will save the output to a JSON file -func ExportJSON(filename string, output []core.Output) error { - exportFilename := fmt.Sprintf("%s.json", filename) - - f, err := os.OpenFile(exportFilename, os.O_CREATE|os.O_WRONLY, 0755) - defer f.Close() - if err != nil { - return err - } - - err = exportJSON(f, output) - if err != nil { - return err - } - log.Info("Results were exported as json in: ", exportFilename) - return nil -} - -func exportJSON(file IFile, output []core.Output) error { - jsonbytes, err := json.Marshal(output) - if err != nil { - return err - } - if _, err := file.WriteString(string(jsonbytes)); err != nil { - return err - } - return nil -} diff --git a/internal/export/export_test.go b/internal/export/export_test.go deleted file mode 100644 index aba7117..0000000 --- a/internal/export/export_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package export - -import ( - "gochopchop/core" - "gochopchop/mock" - "testing" - - "github.com/spf13/afero" -) - -func TestExportCSV(t *testing.T) { - appfs := afero.Afero{Fs: afero.NewMemMapFs()} - filename := "formatcsv" - - var tests = map[string]struct { - output []core.Output - want string - }{ - "correct formatting": {output: mock.FakeOutput, want: mock.FakeOutputAsCSV}, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - f, _ := appfs.Create(filename) - _ = exportCSV(f, tc.output) - contents, _ := appfs.ReadFile(filename) - got := string(contents) - if got != tc.want { - t.Errorf("want : %q, got : %q", tc.want, got) - } - }) - } -} -func TestExportJSON(t *testing.T) { - appfs := afero.Afero{Fs: afero.NewMemMapFs()} - filename := "formatjson" - - var tests = map[string]struct { - output []core.Output - want string - }{ - "correct formatting": {output: mock.FakeOutput, want: mock.FakeOutputAsJSON}, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - f, _ := appfs.Create(filename) - _ = exportJSON(f, tc.output) - contents, _ := appfs.ReadFile(filename) - got := string(contents) - if got != tc.want { - t.Errorf("want : %q, got : %q", tc.want, got) - } - }) - } -} diff --git a/internal/export_test.go b/internal/export_test.go new file mode 100644 index 0000000..3864a00 --- /dev/null +++ b/internal/export_test.go @@ -0,0 +1,260 @@ +package internal_test + +import ( + "io" + "reflect" + "testing" + + "github.com/michelin/gochopchop/internal" +) + +func TestExportersList(t *testing.T) { + t.Parallel() + + expL := internal.ExportersList() + + if expL == "" { + t.Error("Exporters can't be empty") + } +} + +func TestCheckSeverities(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Results []internal.Result + MaxSev internal.Severity + ExpectedErr error + }{ + "invalid-max-severity": { + Results: nil, + MaxSev: -1, + ExpectedErr: &internal.ErrUnsupportedSeverity{-1}, + }, + "invalid-severity": { + Results: []internal.Result{ + {Severity: "invalid-severity"}, + }, + MaxSev: internal.Informational, + ExpectedErr: &internal.ErrInvalidSeverity{"invalid-severity"}, + }, + "reached-severity": { + Results: []internal.Result{ + {Severity: "High"}, + }, + MaxSev: internal.Informational, + ExpectedErr: &internal.ErrMaxSeverityReached{Max: internal.Informational, Sev: internal.High}, + }, + "valid-severities": { + Results: nil, + MaxSev: 0, + ExpectedErr: nil, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + err := internal.CheckSeverities(tt.Results, tt.MaxSev) + + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +type WriteDataCloser interface { + io.WriteCloser + Data() []byte +} + +type FakeWriteCloser struct { + data []byte +} + +func (f *FakeWriteCloser) Write(data []byte) (int, error) { + f.data = append(f.data, data...) + return len(data), nil +} + +func (f *FakeWriteCloser) Close() error { + return nil +} + +func (f *FakeWriteCloser) Data() []byte { + return f.data +} + +func NewFakeWriteCloser(data string) *FakeWriteCloser { + return &FakeWriteCloser{[]byte(data)} +} + +type FailingWriteCloser struct { + io.WriteCloser +} + +func (f *FailingWriteCloser) Write(data []byte) (int, error) { + return 0, errFake +} + +func (f *FailingWriteCloser) Close() error { + return nil +} + +func (f *FailingWriteCloser) Data() []byte { + return nil +} + +func TestExportJSON(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Results []internal.Result + Writer WriteDataCloser + ExpectedOutput []byte + ExpectedErr error + }{ + "nil-results": { + Results: nil, + Writer: &FakeWriteCloser{}, + ExpectedOutput: []byte("null"), // null because Results == nil + ExpectedErr: nil, + }, + "empty-results": { + Results: []internal.Result{}, + Writer: &FakeWriteCloser{}, + ExpectedOutput: []byte("[]"), + ExpectedErr: nil, + }, + "failing-writer": { + Results: nil, + Writer: &FailingWriteCloser{}, + ExpectedOutput: nil, + ExpectedErr: errFake, + }, + "valid": { + Results: []internal.Result{ + { + URL: "https://www.michelin.com/", + Endpoint: "/", + Name: "EXAMPLE", + Severity: "Low", + Remediation: "Remediate", + }, + }, + Writer: &FakeWriteCloser{}, + ExpectedOutput: []byte(`[{"url":"https://www.michelin.com/","endpoint":"/","checkName":"EXAMPLE","severity":"Low","remediation":"Remediate"}]`), + ExpectedErr: nil, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + err := internal.ExportJSON(tt.Results, tt.Writer) + + if !reflect.DeepEqual(tt.Writer.Data(), tt.ExpectedOutput) { + t.Errorf("Failed to get expected output bytes: got \"%v\" instead of \"%v\".", tt.Writer.Data(), tt.ExpectedOutput) + } + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +func TestExportCSV(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Results []internal.Result + Writer WriteDataCloser + ExpectedOutput []byte + ExpectedErr error + }{ + "nil-results": { + Results: nil, + Writer: &FakeWriteCloser{}, + ExpectedOutput: []byte("url,endpoint,severity,checkName,remediation\n"), + ExpectedErr: nil, + }, + "empty-results": { + Results: []internal.Result{}, + Writer: &FakeWriteCloser{}, + ExpectedOutput: []byte("url,endpoint,severity,checkName,remediation\n"), + ExpectedErr: nil, + }, + "failing-writer": { + Results: nil, + Writer: &FailingWriteCloser{}, + ExpectedOutput: nil, + ExpectedErr: errFake, + }, + "valid": { + Results: []internal.Result{ + { + URL: "https://www.michelin.com/", + Endpoint: "/", + Name: "EXAMPLE", + Severity: "Low", + Remediation: "Remediate", + }, + }, + Writer: &FakeWriteCloser{}, + ExpectedOutput: []byte(`url,endpoint,severity,checkName,remediation +https://www.michelin.com/,/,Low,EXAMPLE,Remediate +`), // Notice this \n + ExpectedErr: nil, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + err := internal.ExportCSV(tt.Results, tt.Writer) + + if !reflect.DeepEqual(tt.Writer.Data(), tt.ExpectedOutput) { + t.Errorf("Failed to get expected output bytes: got \"%v\" instead of \"%v\".", tt.Writer.Data(), tt.ExpectedOutput) + } + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +func TestExportTableColor(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Results []internal.Result + Writer WriteDataCloser + ExpectedOutput []byte + ExpectedErr error + }{ + "invalid-severity": { + Results: []internal.Result{ + { + Severity: "invalid-severity", + }, + }, + Writer: &FakeWriteCloser{}, + ExpectedOutput: nil, + ExpectedErr: &internal.ErrInvalidSeverity{"invalid-severity"}, + }, + "full-table": { + Results: []internal.Result{ + {Severity: "High"}, + {Severity: "Medium"}, + {Severity: "Low"}, + {Severity: "Informational"}, + }, + Writer: &FakeWriteCloser{}, + ExpectedOutput: []byte("+-----+----------+---------------+--------+-------------+\n| URL | ENDPOINT | SEVERITY | PLUGIN | REMEDIATION |\n+-----+----------+---------------+--------+-------------+\n| | | \033[31mHigh\033[0m | | |\n| | | \033[32mLow\033[0m | | |\n| | | \033[33mMedium\033[0m | | |\n| | | \033[36mInformational\033[0m | | |\n+-----+----------+---------------+--------+-------------+\n"), + ExpectedErr: nil, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + err := internal.ExportTableColor(tt.Results, tt.Writer) + + if !reflect.DeepEqual(tt.Writer.Data(), tt.ExpectedOutput) { + t.Errorf("Failed to get expected output bytes: got \"%s\" instead of \"%s\".", tt.Writer.Data(), tt.ExpectedOutput) + } + checkErr(err, tt.ExpectedErr, t) + }) + } +} diff --git a/internal/fetch.go b/internal/fetch.go new file mode 100644 index 0000000..f9304c2 --- /dev/null +++ b/internal/fetch.go @@ -0,0 +1,91 @@ +package internal + +import ( + "crypto/tls" + "io" + "net/http" + "time" +) + +// HTTPResponse wraps the status code, body and header +// of a http.Response (it is a simplified version of +// this last). +type HTTPResponse struct { + StatusCode int + Body []byte + Header http.Header +} + +// Fetcher defines the method a http fetcher should define. +type Fetcher interface { + // Get takes an URL and returns a *http.Response from it. + Get(url string) (*http.Response, error) +} + +// Fetcher wraps an IHTTPClient and defines a method to +// fetch an URL. +type NetFetcher struct { + Client http.Client +} + +// NewNetFetcher builds and returns a new fetcher. +func NewNetFetcher(insecure bool, timeout int) NetFetcher { + var tlsCfg *tls.Config + if insecure { + tlsCfg = &tls.Config{InsecureSkipVerify: true} + } + + return NetFetcher{ + Client: http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsCfg, + }, + Timeout: time.Second * time.Duration(timeout), + }, + } +} + +// NewNoRedirectFetcher builds and returns a new +// fetcher that does not follow redirection. +func NewNoRedirectNetFetcher(insecure bool, timeout int) NetFetcher { + var tlsCfg *tls.Config + if insecure { + tlsCfg = &tls.Config{InsecureSkipVerify: true} + } + + return NetFetcher{ + Client: http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsCfg, + }, + Timeout: time.Second * time.Duration(timeout), + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + } +} + +func (nf NetFetcher) Get(url string) (*http.Response, error) { + return nf.Client.Get(url) +} + +// Fetch takes a Fetcher and an URL to return a *HTTPResponse from. +func Fetch(fetcher Fetcher, url string) (*HTTPResponse, error) { + resp, err := fetcher.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return &HTTPResponse{ + Body: body, + StatusCode: resp.StatusCode, + Header: resp.Header, + }, nil +} diff --git a/internal/fetch_test.go b/internal/fetch_test.go new file mode 100644 index 0000000..1aba6df --- /dev/null +++ b/internal/fetch_test.go @@ -0,0 +1,200 @@ +package internal_test + +import ( + "crypto/tls" + "errors" + "io" + "net/http" + "reflect" + "testing" + "time" + + "github.com/michelin/gochopchop/internal" +) + +func TestNewFetcher(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Insecure bool + Timeout int + ExpectedFetcher internal.NetFetcher + }{ + "secure": { + Insecure: false, + Timeout: 0, + ExpectedFetcher: internal.NetFetcher{ + Client: http.Client{ + Transport: &http.Transport{ + TLSClientConfig: nil, + }, + Timeout: time.Duration(0), + }, + }, + }, + "insecure": { + Insecure: true, + Timeout: 0, + ExpectedFetcher: internal.NetFetcher{ + Client: http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Timeout: time.Duration(0), + }, + }, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + f := internal.NewNetFetcher(tt.Insecure, tt.Timeout) + + if tt.Insecure != (f.Client.Transport.(*http.Transport).TLSClientConfig != nil) { + t.Errorf("Failed to get a TLS config appropriate with the insecure parameter (%t): \"%v\".", tt.Insecure, f) + } + if time.Second*time.Duration(tt.Timeout) != f.Client.Timeout { + t.Errorf("Failed to get expected timeout: got \"%v\" instead of \"%v\".", f.Client.Timeout, time.Second*time.Duration(tt.Timeout)) + } + }) + } +} + +func TestNewNoRedirectNetFetcher(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Insecure bool + Timeout int + ExpectedFetcher internal.NetFetcher + }{ + "secure": { + Insecure: false, + Timeout: 0, + ExpectedFetcher: internal.NetFetcher{ + Client: http.Client{ + Transport: &http.Transport{ + TLSClientConfig: nil, + }, + Timeout: time.Duration(0), + }, + }, + }, + "insecure": { + Insecure: true, + Timeout: 0, + ExpectedFetcher: internal.NetFetcher{ + Client: http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Timeout: time.Duration(0), + }, + }, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + f := internal.NewNoRedirectNetFetcher(tt.Insecure, tt.Timeout) + + if tt.Insecure != (f.Client.Transport.(*http.Transport).TLSClientConfig != nil) { + t.Errorf("Failed to get a TLS config appropriate with the insecure parameter (%t): \"%v\".", tt.Insecure, f) + } + if time.Second*time.Duration(tt.Timeout) != f.Client.Timeout { + t.Errorf("Failed to get expected timeout: got \"%v\" instead of \"%v\".", f.Client.Timeout, time.Second*time.Duration(tt.Timeout)) + } + if f.Client.CheckRedirect(nil, nil) != http.ErrUseLastResponse { + t.Error("Failed to set the no-redirection method properly.") + } + }) + } +} + +var michelinHTTPResponse = &internal.HTTPResponse{ + Body: []byte("gochopchop\n"), + StatusCode: 200, + Header: http.Header{ + "Fake-Header": []string{"fake-content"}, + }, +} + +var errFake = errors.New("fake error") + +// FailingReadCloser implements the io.Reader and io.Closer +// interfaces. It will return an error on it's Read method +// call. +type FailingReadCloser struct{} + +func (f *FailingReadCloser) Read([]byte) (int, error) { + return 0, errFake +} + +func (f *FailingReadCloser) Close() error { + return nil +} + +var _ = (io.ReadCloser)(&FailingReadCloser{}) + +// FakeFetcher mocks a Fetcher for the following test. +type FakeFetcher struct{} + +func (f FakeFetcher) Get(url string) (*http.Response, error) { + switch url { + case "https://www.michelin.com/": + return &http.Response{ + Body: NewFakeReadCloser("gochopchop\n"), + StatusCode: 200, + Header: http.Header{ + "Fake-Header": []string{"fake-content"}, + }, + }, nil + case "gochopchop": + return &http.Response{ + Body: &FailingReadCloser{}, + }, nil + default: + return nil, errFake + } +} + +func TestFetch(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Fetcher internal.Fetcher + URL string + ExpectedResp *internal.HTTPResponse + ExpectedErr error + }{ + "valid-url": { + Fetcher: FakeFetcher{}, + URL: "https://www.michelin.com/", + ExpectedResp: michelinHTTPResponse, + ExpectedErr: nil, + }, + "fail-read": { + Fetcher: FakeFetcher{}, + URL: "gochopchop", + ExpectedResp: nil, + ExpectedErr: errFake, + }, + "invalid-url": { + Fetcher: FakeFetcher{}, + URL: "", + ExpectedResp: nil, + ExpectedErr: errFake, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + resp, err := internal.Fetch(tt.Fetcher, tt.URL) + + if !reflect.DeepEqual(resp, tt.ExpectedResp) { + t.Errorf("Failed to get expected HTTPResponse: got \"%v\" instead of \"%v\".", resp, tt.ExpectedResp) + } + checkErr(err, tt.ExpectedErr, t) + }) + } +} diff --git a/internal/formatting/formatting.go b/internal/formatting/formatting.go deleted file mode 100644 index 0e929f1..0000000 --- a/internal/formatting/formatting.go +++ /dev/null @@ -1,44 +0,0 @@ -package formatting - -import ( - "fmt" - "gochopchop/core" - "io" - - "github.com/jedib0t/go-pretty/table" -) - -// PrintTable will render the data as a nice table -func PrintTable(outputs []core.Output, mirror io.Writer) { - colorReset := "\033[0m" - colorRed := "\033[31m" - colorGreen := "\033[32m" - colorYellow := "\033[33m" - colorCyan := "\033[36m" - t := table.NewWriter() - t.SetOutputMirror(mirror) - t.AppendHeader(table.Row{"URL", "Endpoint", "Severity", "Plugin", "Remediation"}) - for _, output := range outputs { - severity := "" - if output.Severity == "High" { - severity = fmt.Sprint(string(colorRed), "High", string(colorReset)) - } else if output.Severity == "Medium" { - severity = fmt.Sprint(string(colorYellow), "Medium", string(colorReset)) - } else if output.Severity == "Low" { - severity = fmt.Sprint(string(colorGreen), "Low", string(colorReset)) - } else { - severity = fmt.Sprint(string(colorCyan), "Informational", string(colorReset)) - } - t.AppendRow([]interface{}{ - output.URL, - output.Endpoint, - severity, - output.Name, - output.Remediation, - }) - } - t.SortBy([]table.SortBy{ - {Name: "Severity", Mode: table.Asc}, - }) - t.Render() -} diff --git a/internal/formatting/formatting_test.go b/internal/formatting/formatting_test.go deleted file mode 100644 index bfb1fc4..0000000 --- a/internal/formatting/formatting_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package formatting_test - -import ( - "bytes" - "gochopchop/internal/formatting" - "gochopchop/mock" - "testing" -) - -func TestFormatOutputTable(t *testing.T) { - mirror := new(bytes.Buffer) - output := mock.FakeOutput - formatting.PrintTable(output, mirror) - got := mirror.String() - want := mock.FakeOutputAsTable - if got != want { - t.Errorf("want : %q, got : %q", want, got) - } -} diff --git a/internal/http.go b/internal/http.go deleted file mode 100644 index 283e8f5..0000000 --- a/internal/http.go +++ /dev/null @@ -1,9 +0,0 @@ -package internal - -import "net/http" - -type HTTPResponse struct { - StatusCode int - Body string - Header http.Header -} diff --git a/internal/httpget/httpget.go b/internal/httpget/httpget.go deleted file mode 100644 index 07436ce..0000000 --- a/internal/httpget/httpget.go +++ /dev/null @@ -1,76 +0,0 @@ -package httpget - -import ( - "crypto/tls" - "gochopchop/internal" - "io/ioutil" - "net/http" - "time" -) - -type IHTTPClient interface { - Get(url string) (*http.Response, error) -} - -type HTTPClient struct { - Transport http.RoundTripper - Timeout time.Duration -} - -type Fetcher struct { - Netclient IHTTPClient -} - -func NewFetcher(insecure bool, timeout int) *Fetcher { - tr := &http.Transport{} - if insecure { - tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - var netClient = &http.Client{ - Transport: tr, - Timeout: time.Second * time.Duration(timeout), - } - return &Fetcher{ - Netclient: netClient, - } -} - -func NewNoRedirectFetcher(insecure bool, timeout int) *Fetcher { - tr := &http.Transport{} - if insecure { - tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - var netClient = &http.Client{ - Transport: tr, - Timeout: time.Second * time.Duration(timeout), - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } - return &Fetcher{ - Netclient: netClient, - } -} - -func (s Fetcher) Fetch(url string) (*internal.HTTPResponse, error) { - - resp, err := s.Netclient.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - bodyString := string(bodyBytes) - - var r = &internal.HTTPResponse{ - Body: bodyString, - StatusCode: resp.StatusCode, - Header: resp.Header, - } - - return r, err -} diff --git a/internal/httpget/httpget_test.go b/internal/httpget/httpget_test.go deleted file mode 100644 index 68597d6..0000000 --- a/internal/httpget/httpget_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package httpget_test - -import ( - "fmt" - "gochopchop/mock" - "testing" -) - -func TestFetch(t *testing.T) { - var tests = map[string]struct { - url string - nilErr bool - }{ - "url return response": {url: "url1", nilErr: true}, - "url return nil response": {url: "unknown", nilErr: false}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - resp, err := mock.FakeFetcher.Fetch(tc.url) - fmt.Printf("%v - %v \n", resp, err) - if tc.nilErr && err != nil { - t.Errorf("expected a nil error, got : %v", err) - } - if !tc.nilErr && err == nil { - t.Errorf("expected a non-nil error, got : %v", err) - } - }) - } -} diff --git a/internal/result.go b/internal/result.go new file mode 100644 index 0000000..84309aa --- /dev/null +++ b/internal/result.go @@ -0,0 +1,55 @@ +package internal + +import ( + "sort" + "sync" +) + +// Result wraps result for each finding of the scan. +type Result struct { + URL string `json:"url"` + Endpoint string `json:"endpoint"` + Name string `json:"checkName"` + Severity string `json:"severity"` + Remediation string `json:"remediation"` +} + +// SafeResults stores a Result slice. +type SafeResults struct { + mux sync.Mutex + Res []Result +} + +// Append adds a Result to the existing ones. Does not +// check for duplications. +func (s *SafeResults) Append(res Result) { + s.mux.Lock() + defer s.mux.Unlock() + + s.Res = append(s.Res, res) +} + +// GetResults returns the Result array of SafeResults. +func (s *SafeResults) GetResults() ResultSlice { + return s.Res +} + +// ResultSlice wraps a Result slice to enable sorting it. +type ResultSlice []Result + +func (rs ResultSlice) Len() int { + return len(rs) +} + +func (rs ResultSlice) Less(i, j int) bool { + if rs[i].URL == rs[j].URL { + return rs[i].Endpoint < rs[j].Endpoint + } + return rs[i].URL < rs[j].URL +} + +func (rs ResultSlice) Swap(i, j int) { + rs[i], rs[j] = rs[j], rs[i] +} + +var _ = (sort.Interface)(ResultSlice{}) diff --git a/internal/result_test.go b/internal/result_test.go new file mode 100644 index 0000000..5d52d31 --- /dev/null +++ b/internal/result_test.go @@ -0,0 +1,87 @@ +package internal_test + +import ( + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/michelin/gochopchop/internal" +) + +func TestSafeResultsAppend(t *testing.T) { + t.Parallel() + + s := internal.SafeResults{} + s.Append(internal.Result{}) + + if !cmp.Equal(s.Res, []internal.Result{{}}) { + t.Error("Failed to properly add a Result in the SafeResults.") + } +} +func TestSafeResultsGetResults(t *testing.T) { + t.Parallel() + + resSlice := internal.ResultSlice{ + { + URL: "test", + Endpoint: "/", + Name: "name", + Severity: "Informational", + Remediation: "remediation", + }, + } + + s := internal.SafeResults{ + Res: resSlice, + } + + res := s.GetResults() + + if !cmp.Equal(res, resSlice) { + t.Errorf("Failed to get expected results: \"%v\" instead of \"%v\".", res, resSlice) + } +} + +func TestResultSliceSort(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + ResultSlice internal.ResultSlice + ExpectedResultSlice internal.ResultSlice + }{ + "empty-slice": { + ResultSlice: internal.ResultSlice{}, + ExpectedResultSlice: internal.ResultSlice{}, + }, + "not-empty-slice": { + ResultSlice: internal.ResultSlice{ + { + URL: "http://127.0.0.1:8080", + Endpoint: "/", + }, { + URL: "http://127.0.0.1:8080", + Endpoint: "/2", + }, + }, + ExpectedResultSlice: internal.ResultSlice{ + { + URL: "http://127.0.0.1:8080", + Endpoint: "/", + }, { + URL: "http://127.0.0.1:8080", + Endpoint: "/2", + }, + }, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + sort.Stable(tt.ResultSlice) + + if !cmp.Equal(tt.ResultSlice, tt.ExpectedResultSlice) { + t.Errorf("Failed to sort as expected: got \"%v\" instead of \"%v\".", tt.ResultSlice, tt.ExpectedResultSlice) + } + }) + } +} diff --git a/internal/scan.go b/internal/scan.go new file mode 100644 index 0000000..23f4c6b --- /dev/null +++ b/internal/scan.go @@ -0,0 +1,352 @@ +package internal + +import ( + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// Scanner defines the method a scanner must implement. +type Scanner interface { + Run(urls []string, doneChan <-chan struct{}) ([]Result, error) +} + +// Scan is the entrypoint of the scan process. +func Scan(scanner Scanner, urls []string, doneChan <-chan struct{}) (ResultSlice, time.Duration, error) { + begin := time.Now() + res, err := scanner.Run(urls, doneChan) + if err != nil { + return nil, time.Duration(0), err + } + + return res, time.Since(begin), nil +} + +// Scanner wraps the Signatures and the fetchers. +// +// Two fetchers are needed because we can't use the same +// http client to follow redirects and not to. +type CoreScanner struct { + Signatures *Signatures + Fetcher Fetcher + NoRedirectFetcher Fetcher + Goroutines int + SafeResults *SafeResults +} + +var _ = (Scanner)(&CoreScanner{}) + +func NewCoreScanner(config *Config, signatures *Signatures) (*CoreScanner, error) { + // Validate parameters + if config == nil { + return nil, &ErrNilParameter{"config"} + } + if signatures == nil { + return nil, &ErrNilParameter{"signatures"} + } + + return &CoreScanner{ + Signatures: signatures, + Fetcher: NewNetFetcher(config.HTTP.Insecure, config.HTTP.Timeout), + NoRedirectFetcher: NewNoRedirectNetFetcher(config.HTTP.Insecure, config.HTTP.Timeout), + SafeResults: &SafeResults{ + Res: []Result{}, + mux: sync.Mutex{}, + }, + Goroutines: config.Goroutines, + }, nil +} + +type workerJob struct { + url string + endpoint string + plugin *Plugin +} + +// RunScan scans the urls until job is completed or +// a done signal is sent throuh the chan. +func (scanner *CoreScanner) Run(urls []string, doneChan <-chan struct{}) ([]Result, error) { + // Split the load in channels to work on concurrently + workJobs := splitWork(scanner.Goroutines, urls, scanner.Signatures.Plugins) + workJobsChan := channelize(workJobs) + + var wgJobs sync.WaitGroup + for _, wj := range workJobsChan { + wgJobs.Add(1) + go func(wj chan workerJob) { + defer wgJobs.Done() + for { + select { + case <-doneChan: + return + + default: + return + + case job := <-wj: + // Fetch the HTTP response from url + resp, err := scanner.Fetch(job.url+job.endpoint, job.plugin.FollowRedirects) + if err != nil { + logrus.Error(err) + break + } + + // Procede to checks concurrently + var swg sync.WaitGroup + for _, check := range job.plugin.Checks { + swg.Add(1) + go func(check Check) { + defer swg.Done() + select { + case <-doneChan: + return + + default: + match, err := check.Match(resp) + if err != nil { + return + } + if match { + scanner.SafeResults.Append(Result{ + URL: job.url, + Name: check.Name, + Endpoint: job.endpoint, + Severity: check.Severity, + Remediation: check.Remediation, + }) + } + } + }(check) + } + swg.Wait() + } + } + }(wj) + } + + wgJobs.Wait() + for _, wj := range workJobsChan { + close(wj) + } + + return scanner.SafeResults.Res, nil +} + +// ErrNilFetcher is an error meaning a CoreScanner fetcher +// is nil. +type ErrNilFetcher struct { + FetcherName string +} + +func (e ErrNilFetcher) Error() string { + return e.FetcherName + " is nil" +} + +// Fetch fetches content from an URL from its fetchers +// with or without redirection. +func (s CoreScanner) Fetch(url string, followRedirects bool) (*HTTPResponse, error) { + // Redirect + if followRedirects { + if s.Fetcher == nil { + return nil, &ErrNilFetcher{"Fetcher"} + } + return Fetch(s.Fetcher, url) + } + + // No-redirect + if s.NoRedirectFetcher == nil { + return nil, &ErrNilFetcher{"NoRedirectFetcher"} + } + return Fetch(s.NoRedirectFetcher, url) +} + +// This is the implementation of the following algorithm, which aims to +// split the work into pieces, in near-equal parts, with a bit of optimization +// to avoid working on empty stuff. +// +// Let's have a set to split in n parts. How to tend to a prefectly-splitted +// load from this ? Dividing it seems to be a good solution. +// +// In our case, we are having endpoints and threads (it will be implemented on +// goroutines instead of threads, so only keep the idea behind). +// \#endpoints = \#urls \times \sum_{p=0}^{\#plugins} \#endpoints_p +// +// To split the entrypoints to subsets (integer-long), we divide it by our number +// of subsets, e.g. number of threads, which gives us the length of a "standard" +// subset. The remaining goes into a last subset. +// L_s stands for "standard length" ; L_r stands for "remaining length". +// L_s = \left \lceil \frac{\#endpoints}{\#threads} \right \rceil +// L_r = \#endpoints - L_s \times (\#threads - 1) +// Notice L_s \geq L_r. +// +// We now need to define how many subset of length L_s and L_r will be built, +// and by construction we know the following. +// \#L_s = \#threads - 1 +// \#L_r = 1 +// +// Examples: +// - Case 16/4: +// \#endpoints = 16 ; \#threads = 4 +// L_s = ceil(16/4) = 4 +// L_r = 16 - 4*(4-1) = 4 +// We can represent it as follows. +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// 0 1 2 3 +// - Case 21/4: +// \#endpoints = 21 ; \#threads = 4 +// L_s = ceil(21/4) = 6 +// L_r = 21 - 6*(4-1) = 3 +// We can represent it as follows. +// ■ ■ ■ +// ■ ■ ■ +// ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// 0 1 2 3 +// +// We can see that the last one is taking a nap while the other are still +// working. +// nap = L_s - L_r +// So, we want to move "toFill" endpoints to the last subset. +// toFill = nap - 1 = L_s - L_r - 1 +// +// Let's change our concept and talk of "filled" and "reduced" subsets. +// To avoid having the last taking a nap while the other are working, we "move" last +// endpoint of toFill "standard" subsets to this last one, so we have the following. +// \#reduced = toFill + 1 +// \#filled = \#threads - \#reduced = \#threads - (toFill + 1) = \#threads - toFill - 1 +// +// Then, by construction, we define L_f (filled length) and L_r (reduced length) as follows. +// L_f = \left \lceil \frac{\#endpoints}{\#threads} \right \rceil +// L_r = \left \lfloor \frac{\#endpoints}{\#threads} \right \rfloor +// +// Now, to avoid relaying on the previous step to achieve this one, we develop and reduce. +// \#reduced = toFill + 1 +// = L_s - L_r - 1 (L_r from the previous algorithm) +// = \left \lceil \frac{\#endpoints}{\#threads} \right \rceil - (\#endpoints - \left \lceil \frac{\#endpoints}{\#threads} \right \rceil \times (\#threads - 1)) +// = \left \lceil \frac{\#endpoints}{\#threads} \right \rceil - \#endpoints + \left \lceil \frac{\#endpoints}{\#threads} \right \rceil \times (\#threads - 1) +// = \left \lceil \frac{\#endpoints}{\#threads} \right \rceil \times \#threads - \#endpoints +// +// Examples: +// - Case 16/4: +// \#endpoints = 16 ; \#threads = 4 +// L_f = ceil(16/4) = 4 +// L_r = floor(16/4) = 4 +// #reduced = 4 * 4 - 16 = 0 +// #filled = 4 - 0 = 4 +// We can represent it as follows. +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// 0 1 2 3 +// - Case 21/4: +// \#endpoints = 21 ; \#threads = 4 +// L_f = ceil(21/4) = 6 +// L_r = floor(21/4) = 5 +// #reduced = 6 * 4 - 21 = 3 +// #filled = 4 - 3 = 1 +// We can represent it as follows. +// ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// ■ ■ ■ ■ +// 0 1 2 3 +// +// For the first version, in case L_s == \#threads - 1 (worst case) this last +// optimization avoids the last worker to work only on one job (see it with +// the case 157/14 where 13 works on 12 jobs and the last on only one). +func splitWork(pieces int, urls []string, plugins []Plugin) [][]workerJob { + // Compute #endpoints + var n_endpoints int + for _, plugin := range plugins { + n_endpoints += len(plugin.Endpoints) + } + n_endpoints *= len(urls) + + // Compute L_filled and L_reduced + l_filled := n_endpoints / pieces + l_reduced := l_filled + if l_filled*pieces != n_endpoints { + // Ceil l_filled + l_filled++ + } + + // Compute #reduced and #filled + n_reduced := l_filled*pieces - n_endpoints + n_filled := pieces - n_reduced + + // Avoid having empty work groups + if l_filled == 0 { + return [][]workerJob{} + } + if l_reduced == 0 { + n_reduced = 0 + } + + // Build work groups + n_total := n_filled + n_reduced + wg := make([][]workerJob, n_total) + for i := 0; i < n_filled; i++ { + wg[i] = make([]workerJob, l_filled) + } + for i := 0; i < n_reduced; i++ { + wg[n_filled+i] = make([]workerJob, l_reduced) + } + + // Load balance + indexWG := 0 // Index of the work group + indexInCurrWG := 0 // Index in the current work group + lenCurrWg := l_filled // Length of the current work group + currWG := &wg[0] // Place iterator at first work group + lp := len(plugins) + for _, url := range urls { + for i := 0; i < lp; i++ { + for _, endp := range plugins[i].Endpoints { + // Set the work group endpoint + (*currWG)[indexInCurrWG] = workerJob{ + url: url, + endpoint: endp, + plugin: &plugins[i], + } + indexInCurrWG++ + + // Move the indexes if needed. + if indexInCurrWG == lenCurrWg { + indexInCurrWG = 0 + indexWG++ + + if indexWG == n_filled { + lenCurrWg = l_reduced + } + + if indexWG != n_total { + // Move to the next only if not the last in slice + currWG = &wg[indexWG] + } + } + } + } + } + + return wg +} + +func channelize(workerJobs [][]workerJob) []chan workerJob { + chans := make([]chan workerJob, len(workerJobs)) + for i, wj := range workerJobs { + chans[i] = make(chan workerJob, len(wj)) + for _, w := range wj { + chans[i] <- w + } + } + + return chans +} diff --git a/internal/scan_test.go b/internal/scan_test.go new file mode 100644 index 0000000..1203fa6 --- /dev/null +++ b/internal/scan_test.go @@ -0,0 +1,174 @@ +package internal_test + +import ( + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/michelin/gochopchop/internal" +) + +// FakeScanner mocks an internal.Scanner. +type FakeScanner struct{} + +func (f *FakeScanner) Run(urls []string, doneChan <-chan struct{}) ([]internal.Result, error) { + var res []internal.Result + for _, url := range urls { + switch url { + case "https://www.michelin.com/": + res = append(res, internal.Result{ + URL: url, + Endpoint: "/", + Name: "michelin", + Severity: "Low", + Remediation: "Work on open-sources projects.", + }) + default: + return nil, errFake + } + } + return res, nil +} + +var _ = (internal.Scanner)(&FakeScanner{}) + +func TestScan(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Scanner internal.Scanner + URLs []string + DoneChan <-chan struct{} + ExpectedResultSlice internal.ResultSlice + ExpectedErr error + }{ + "success": { + Scanner: &FakeScanner{}, + URLs: []string{"https://www.michelin.com/"}, + DoneChan: nil, + ExpectedResultSlice: internal.ResultSlice{{ + URL: "https://www.michelin.com/", + Endpoint: "/", + Name: "michelin", + Severity: "Low", + Remediation: "Work on open-sources projects.", + }}, + ExpectedErr: nil, + }, + "failure": { + Scanner: &FakeScanner{}, + URLs: []string{""}, + DoneChan: nil, + ExpectedResultSlice: nil, + ExpectedErr: errFake, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + resp, _, err := internal.Scan(tt.Scanner, tt.URLs, tt.DoneChan) + + if !cmp.Equal(resp, tt.ExpectedResultSlice) { + t.Errorf("Failed to get expected ResultSlice: got \"%v\" instead of \"%v\".", resp, tt.ExpectedResultSlice) + } + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +func TestNewCoreScanner(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Config *internal.Config + Signatures *internal.Signatures + ExpectedCoreScanner *internal.CoreScanner + ExpectedErr error + }{ + "nil-config": { + Config: nil, + Signatures: nil, + ExpectedCoreScanner: nil, + ExpectedErr: &internal.ErrNilParameter{"config"}, + }, + "nil-signatures": { + Config: &internal.Config{}, + Signatures: nil, + ExpectedCoreScanner: nil, + ExpectedErr: &internal.ErrNilParameter{"signatures"}, + }, + "core-scanner": { + Config: &internal.Config{}, + Signatures: &internal.Signatures{}, + ExpectedCoreScanner: &internal.CoreScanner{}, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + scan, err := internal.NewCoreScanner(tt.Config, tt.Signatures) + + if (scan == nil) != (tt.ExpectedCoreScanner == nil) { + t.Errorf("Failed to get a non-nil CoreScanner: got \"%v\" instead of \"%v\".", scan, tt.ExpectedCoreScanner) + } + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +func TestCoreScannerFetch(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + CoreScanner internal.CoreScanner + URL string + FollowRedirects bool + ExpectedResp *internal.HTTPResponse + ExpectedErr error + }{ + "no-redirect": { + CoreScanner: internal.CoreScanner{ + NoRedirectFetcher: FakeFetcher{}, + }, + URL: "https://www.michelin.com/", + FollowRedirects: false, + ExpectedResp: michelinHTTPResponse, + ExpectedErr: nil, + }, + "nil-no-redirect": { + CoreScanner: internal.CoreScanner{}, + URL: "https://www.michelin.com/", + FollowRedirects: false, + ExpectedResp: nil, + ExpectedErr: &internal.ErrNilFetcher{"NoRedirectFetcher"}, + }, + "redirect": { + CoreScanner: internal.CoreScanner{ + Fetcher: FakeFetcher{}, + }, + URL: "https://www.michelin.com/", + FollowRedirects: true, + ExpectedResp: michelinHTTPResponse, + ExpectedErr: nil, + }, + "nil-redirect": { + CoreScanner: internal.CoreScanner{}, + URL: "https://www.michelin.com/", + FollowRedirects: true, + ExpectedResp: nil, + ExpectedErr: &internal.ErrNilFetcher{"Fetcher"}, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + resp, err := tt.CoreScanner.Fetch(tt.URL, tt.FollowRedirects) + + if !reflect.DeepEqual(resp, tt.ExpectedResp) { + t.Errorf("Failed to get expected HTTPResponse: got \"%v\" instead of \"%v\".", resp, tt.ExpectedResp) + } + + checkErr(err, tt.ExpectedErr, t) + }) + } +} diff --git a/internal/severity.go b/internal/severity.go new file mode 100644 index 0000000..81b9f2a --- /dev/null +++ b/internal/severity.go @@ -0,0 +1,74 @@ +package internal + +import ( + "strconv" +) + +// Severity is a custom type defining a Signature +// severity. +// This custom type enables direct comparison between +// severities. +type Severity int + +const ( + High Severity = iota + Medium + Low + Informational + + highKey = "High" + mediumKey = "Medium" + lowKey = "Low" + infoKey = "Informational" +) + +// ErrInvalidSeverity is an error meaning a given +// severity is not supported. +type ErrInvalidSeverity struct { + Severity string +} + +func (e ErrInvalidSeverity) Error() string { + return e.Severity + " is not a valid severity" +} + +// ErrUnsupportedSeverity is an error meaning a severity +// is not supported, depending on the context. +type ErrUnsupportedSeverity struct { + Severity Severity +} + +func (e ErrUnsupportedSeverity) Error() string { + return "unsupported severity: " + strconv.Itoa(int(e.Severity)) +} + +func (s Severity) String() (string, error) { + switch s { + case High: + return highKey, nil + case Medium: + return mediumKey, nil + case Low: + return lowKey, nil + case Informational: + return infoKey, nil + default: + return "", &ErrUnsupportedSeverity{s} + } +} + +// StringToSeverity converts a Severity to its string. +func StringToSeverity(severity string) (Severity, error) { + switch severity { + case highKey: + return High, nil + case mediumKey: + return Medium, nil + case lowKey: + return Low, nil + case infoKey: + return Informational, nil + default: + return 0, &ErrInvalidSeverity{severity} + } +} diff --git a/internal/severity_test.go b/internal/severity_test.go new file mode 100644 index 0000000..626ca2a --- /dev/null +++ b/internal/severity_test.go @@ -0,0 +1,103 @@ +package internal_test + +import ( + "testing" + + "github.com/michelin/gochopchop/internal" +) + +func TestSeverityString(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Severity internal.Severity + ExpectedStr string + ExpectedErr error + }{ + "high": { + Severity: internal.High, + ExpectedStr: "High", + ExpectedErr: nil, + }, + "medium": { + Severity: internal.Medium, + ExpectedStr: "Medium", + ExpectedErr: nil, + }, + "low": { + Severity: internal.Low, + ExpectedStr: "Low", + ExpectedErr: nil, + }, + "info": { + Severity: internal.Informational, + ExpectedStr: "Informational", + ExpectedErr: nil, + }, + "invalid": { + Severity: -1, + ExpectedStr: "", + ExpectedErr: &internal.ErrUnsupportedSeverity{-1}, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + str, err := tt.Severity.String() + + if str != tt.ExpectedStr { + t.Error("Failed to get expected Severity.String() result: got \"" + str + "\" instead of \"" + tt.ExpectedStr) + } + + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +func TestStringToSeverity(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Severity string + ExpectedSeverity internal.Severity + ExpectedErr error + }{ + "high": { + Severity: "High", + ExpectedSeverity: internal.High, + ExpectedErr: nil, + }, + "medium": { + Severity: "Medium", + ExpectedSeverity: internal.Medium, + ExpectedErr: nil, + }, + "low": { + Severity: "Low", + ExpectedSeverity: internal.Low, + ExpectedErr: nil, + }, + "info": { + Severity: "Informational", + ExpectedSeverity: internal.Informational, + ExpectedErr: nil, + }, + "invalid": { + Severity: "invalid", + ExpectedSeverity: 0, + ExpectedErr: &internal.ErrInvalidSeverity{"invalid"}, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + sev, err := internal.StringToSeverity(tt.Severity) + + if sev != tt.ExpectedSeverity { + t.Error("Failed to get expected severity: got \"", sev, "\" instead of \"", tt.ExpectedSeverity, "\"") + } + + checkErr(err, tt.ExpectedErr, t) + }) + } +} diff --git a/internal/signatures.go b/internal/signatures.go new file mode 100644 index 0000000..6c7ce96 --- /dev/null +++ b/internal/signatures.go @@ -0,0 +1,250 @@ +package internal + +import ( + "bytes" + "errors" + "io" + "os" + "strings" + + "github.com/jedib0t/go-pretty/table" + "gopkg.in/yaml.v2" +) + +// Signatures represents the plugins/rules from the +// .yaml configuration file. It's the root of a config +// file. +type Signatures struct { + Plugins []Plugin `yaml:"plugins"` +} + +// Plugin means an entry to test for during scan. +type Plugin struct { + Endpoints []string `yaml:"endpoints"` + Checks []Check `yaml:"checks"` + FollowRedirects bool `yaml:"follow_redirects"` +} + +// Check is a check the scan runs in. +type Check struct { + MustMatchOne []string `yaml:"match"` + MustMatchAll []string `yaml:"all_match"` + MustNotMatch []string `yaml:"no_match"` + StatusCode *int `yaml:"status_code"` + Name string `yaml:"name"` + Remediation string `yaml:"remediation"` + Severity string `yaml:"severity"` + Description string `yaml:"description"` + Headers []string `yaml:"headers"` + NoHeaders []string `yaml:"no_headers"` +} + +// ErrInvalidHeaderFormat is an error meaning an header +// format is invalid. +type ErrInvalidHeaderFormat struct { + Header string +} + +func (e ErrInvalidHeaderFormat) Error() string { + return "invalid header format: " + e.Header + " should be \"KEY:VALUE\"" +} + +// Match analyses the HTTP Response. A match means that +// one of the criteria has been met (through the strategies +// of MatchAll/MatchOne/NotMatch, and Headers/NotHeaders). +func (check *Check) Match(resp *HTTPResponse) (bool, error) { + // Test nils + if check == nil { + return false, &ErrNilParameter{"check"} + } + if check.StatusCode == nil { + return false, &ErrNilParameter{"check.StatusCode"} + } + if resp == nil { + return false, &ErrNilParameter{"resp"} + } + + // Test status code + if resp.StatusCode != *check.StatusCode { + return false, nil + } + + // Check for MatchAll + for _, match := range check.MustMatchAll { + if !bytes.Contains(resp.Body, []byte(match)) { + return false, nil + } + } + + // Check for MatchOne + found := false + for _, match := range check.MustMatchOne { + if bytes.Contains(resp.Body, []byte(match)) { + found = true + break + } + } + if !found { + return false, nil + } + + // Check for NotMatch + for _, match := range check.MustNotMatch { + if bytes.Contains(resp.Body, []byte(match)) { + return false, nil + } + } + + // Check for headers + for _, header := range check.Headers { + hs := strings.Split(header, ":") + if len(hs) != 2 { + return false, &ErrInvalidHeaderFormat{header} + } + hKey := hs[0] + hVal := hs[1] + + // Check for header in the HTTPResponse by its key + respHVal, ok := resp.Header[hKey] + if !ok { + return false, nil + } + + // Look for a match + found = false + for _, respHeaderValue := range respHVal { + if strings.Contains(respHeaderValue, hVal) { + found = true + break + } + } + if !found { + return false, nil + } + } + + // Check for NoHeaders + for _, header := range check.NoHeaders { + pNH := strings.Split(header, ":") + if len(pNH) != 2 { + return false, &ErrInvalidHeaderFormat{header} + } + + nhKey := pNH[0] + nhVal := pNH[1] + if respHeaderValues, kFound := resp.Header[nhKey]; kFound { + vFound := false + for _, respHeaderValue := range respHeaderValues { + if strings.Contains(respHeaderValue, nhVal) { + vFound = true + break + } + } + if vFound { + return false, nil + } + } + } + + // If matches everything, then it's fine + return true, nil +} + +// ErrCheckInvalidField is an error meaning a check +// field is invalid. +type ErrCheckInvalidField struct { + Check string + Field string +} + +func (e ErrCheckInvalidField) Error() string { + return "missing or empty " + e.Field + " in " + e.Check + " plugin checks." +} + +// ErrInvalidPathSignaturesFile is an error meaning +// that the path to the signatures file is invalid. +var ErrInvalidPathSignaturesFile = errors.New("path of signatures file is not valid") + +// ErrBothEndpointSet is an error meaning endpoint and +// endpoints are set at same time. +var ErrBothEndpointSet = errors.New("URI and URIs can't be set at the same time in plugin checks") + +func ReaderFromFile(path string) (io.Reader, error) { + // Check signature file exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil, ErrInvalidPathSignaturesFile + } + + // Open file + signFile, err := os.Open(path) + if err != nil { + return nil, err + } + + return signFile, nil +} + +// ParseSignatures parses and returns the signatures +// from the path of the file containg those. +func ParseSignatures(r io.Reader) (*Signatures, error) { + // Read its content + signData, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + // Build signatures + var sign Signatures + err = yaml.Unmarshal(signData, &sign) + if err != nil { + return nil, err + } + + // Validate plugins + for _, plugin := range sign.Plugins { + // Ensure the plugin's checks content are valid + for _, check := range plugin.Checks { + // Check main fields are not empty + switch "" { + case check.Description: + return nil, &ErrCheckInvalidField{Check: check.Name, Field: "description"} + case check.Remediation: + return nil, &ErrCheckInvalidField{Check: check.Name, Field: "remediation"} + case check.Severity: + return nil, &ErrCheckInvalidField{Check: check.Name, Field: "severity"} + } + + // Check severity is valid + if _, err := StringToSeverity(check.Severity); err != nil { + return nil, err + } + + // Check headers to ensure they match KEY:VALUE fmt + for _, header := range check.Headers { + if strings.Count(header, ":") != 1 { + return nil, &ErrInvalidHeaderFormat{header} + } + } + } + } + + return &sign, nil +} + +// PrintSignatures prints the sign with the save severity as the sevStr. +func PrintSignatures(sign *Signatures, sevStr string, w io.Writer) { + cpt := 0 + t := table.NewWriter() + t.SetOutputMirror(w) + t.AppendHeader(table.Row{"Endpoint", "Check Name", "Severity", "Description"}) + for _, plugin := range sign.Plugins { + for _, check := range plugin.Checks { + if sevStr == check.Severity { + t.AppendRow([]interface{}{plugin.Endpoints, check.Name, check.Severity, check.Description}) + cpt++ + } + } + } + t.AppendFooter(table.Row{"", "", "Total Checks", cpt}) + t.Render() +} diff --git a/internal/signatures_test.go b/internal/signatures_test.go new file mode 100644 index 0000000..ea0d3f6 --- /dev/null +++ b/internal/signatures_test.go @@ -0,0 +1,453 @@ +package internal_test + +import ( + "io" + "net/http" + "reflect" + "testing" + + "github.com/michelin/gochopchop/internal" + "gopkg.in/yaml.v2" +) + +var ( + statuscode1 int = 1 + statuscode2 int = 2 + + triggerStr string = "trigger" + triggerBts []byte = []byte(triggerStr) +) + +func TestCheckMatch(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Check *internal.Check + Resp *internal.HTTPResponse + ExpectedBool bool + ExpectedErr error + }{ + "nil-check": { + Check: nil, + Resp: nil, + ExpectedBool: false, + ExpectedErr: &internal.ErrNilParameter{"check"}, + }, + "nil-check-statuscode": { + Check: &internal.Check{ + StatusCode: nil, + }, + Resp: nil, + ExpectedBool: false, + ExpectedErr: &internal.ErrNilParameter{"check.StatusCode"}, + }, + "nil-resp": { + Check: &internal.Check{ + StatusCode: &statuscode1, + }, + Resp: nil, + ExpectedBool: false, + ExpectedErr: &internal.ErrNilParameter{"resp"}, + }, + "different-check-statuscode": { + Check: &internal.Check{ + StatusCode: &statuscode1, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode2, + }, + ExpectedBool: false, + ExpectedErr: nil, + }, + "must-match-all-false": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchAll: []string{triggerStr}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: []byte{}, + }, + ExpectedBool: false, + ExpectedErr: nil, + }, + "must-match-one-not-found": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: []byte{}, + }, + ExpectedBool: false, + ExpectedErr: nil, + }, + "must-match-one-found": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + }, + ExpectedBool: true, + ExpectedErr: nil, + }, + "must-not-match-false": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + MustNotMatch: []string{triggerStr}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + }, + ExpectedBool: false, + ExpectedErr: nil, + }, + "invalid-header-format": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + Headers: []string{"Fake-Header:first:second"}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + }, + ExpectedBool: false, + ExpectedErr: &internal.ErrInvalidHeaderFormat{"Fake-Header:first:second"}, + }, + "unknown-header": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + Headers: []string{"Fake-Header:fake-content"}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + Header: http.Header{}, + }, + ExpectedBool: false, + ExpectedErr: nil, + }, + "valid-match-header": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + Headers: []string{"Fake-Header:fake-content"}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + Header: http.Header{ + "Fake-Header": []string{"fake-content"}, + }, + }, + ExpectedBool: true, + ExpectedErr: nil, + }, + "not-match-header": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + Headers: []string{"Fake-Header:fake-content"}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + Header: http.Header{ + "Fake-Header": []string{"invalid-content"}, + }, + }, + ExpectedBool: false, + ExpectedErr: nil, + }, + "invalid-no-header-format": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + NoHeaders: []string{"Fake-Header:first:second"}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + }, + ExpectedBool: false, + ExpectedErr: &internal.ErrInvalidHeaderFormat{"Fake-Header:first:second"}, + }, + "match-no-header": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + NoHeaders: []string{"Fake-Header:fake-content"}, + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + Header: http.Header{ + "Fake-Header": []string{"fake-content"}, + }, + }, + ExpectedBool: false, + ExpectedErr: nil, + }, + "valid-check": { + Check: &internal.Check{ + StatusCode: &statuscode1, + MustMatchOne: []string{triggerStr}, // To pass the MatchOne check + }, + Resp: &internal.HTTPResponse{ + StatusCode: statuscode1, + Body: triggerBts, + }, + ExpectedBool: true, + ExpectedErr: nil, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + b, err := tt.Check.Match(tt.Resp) + + if b != tt.ExpectedBool { + t.Errorf("Failed to get expected bool value: got \"%t\" instead of \"%t\".", b, tt.ExpectedBool) + } + + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +func TestParseSignatures(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Reader io.Reader + ExpectedSign *internal.Signatures + ExpectedErr error + }{ + "fail-reader": { + Reader: &FailingReadCloser{}, + ExpectedSign: nil, + ExpectedErr: errFake, + }, + "invalid-signatures": { + Reader: NewFakeReadCloser("invalid-content"), + ExpectedSign: nil, + ExpectedErr: &yaml.TypeError{}, + }, + "empty-description": { + Reader: NewFakeReadCloser(` +insecure: false +plugins: + - endpoints: + - "example.html" + checks: + - name: EXAMPLE + match: + - "Example:" + remediation: Remediation example + description: + severity: Informational`), + ExpectedSign: nil, + ExpectedErr: &internal.ErrCheckInvalidField{"EXAMPLE", "description"}, + }, + "empty-remediation": { + Reader: NewFakeReadCloser(` +insecure: false +plugins: + - endpoints: + - "example.html" + checks: + - name: EXAMPLE + match: + - "Example:" + remediation: + description: Remediation example + severity: Informational`), + ExpectedSign: nil, + ExpectedErr: &internal.ErrCheckInvalidField{"EXAMPLE", "remediation"}, + }, + "empty-severity": { + Reader: NewFakeReadCloser(` +insecure: false +plugins: + - endpoints: + - "example.html" + checks: + - name: EXAMPLE + match: + - "Example:" + remediation: Description example + description: Remediation example + severity:`), + ExpectedSign: nil, + ExpectedErr: &internal.ErrCheckInvalidField{"EXAMPLE", "severity"}, + }, + "invalid-severity": { + Reader: NewFakeReadCloser(` +insecure: false +plugins: + - endpoints: + - "example.html" + checks: + - name: EXAMPLE + match: + - "Example:" + remediation: Description example + description: Remediation example + severity: INVALID`), + ExpectedSign: nil, + ExpectedErr: &internal.ErrInvalidSeverity{"INVALID"}, + }, + "invalid-header-format": { + Reader: NewFakeReadCloser(` +insecure: false +plugins: + - endpoints: + - "example.html" + checks: + - name: EXAMPLE + match: + - "Example:" + remediation: Description example + description: Remediation example + severity: Low + headers: + - "Fake-Header:fake:content"`), + ExpectedSign: nil, + ExpectedErr: &internal.ErrInvalidHeaderFormat{Header: "Fake-Header:fake:content"}, + }, + "empty-valid-signatures": { + Reader: NewFakeReadCloser(""), + ExpectedSign: &internal.Signatures{}, + ExpectedErr: nil, + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + sign, err := internal.ParseSignatures(tt.Reader) + + if !reflect.DeepEqual(sign, tt.ExpectedSign) { + t.Errorf("Failed to get expected *Signatures: got \"%v\" instead of \"%v\".", sign, tt.ExpectedSign) + } + checkErr(err, tt.ExpectedErr, t) + }) + } +} + +type WriteData interface { + io.Writer + Data() []byte +} + +type FakeWriter struct { + data []byte +} + +func (f *FakeWriter) Write(data []byte) (int, error) { + f.data = append(f.data, data...) + return len(data), nil +} + +func (f *FakeWriter) Data() []byte { + return f.data +} + +func NewFakeWriter(data string) *FakeWriter { + return &FakeWriter{[]byte(data)} +} + +var _ = (io.Writer)(&FakeWriter{}) +var _ = (WriteData)(&FakeWriter{}) + +func TestPrintSignatures(t *testing.T) { + t.Parallel() + + var tests = map[string]struct { + Writer WriteData + Sign *internal.Signatures + Sev string + ExpectedOutput []byte + }{ + "empty-sign": { + Writer: NewFakeWriter(""), + Sign: &internal.Signatures{}, + Sev: "", + ExpectedOutput: []byte(`+----------+------------+--------------+-------------+ +| ENDPOINT | CHECK NAME | SEVERITY | DESCRIPTION | ++----------+------------+--------------+-------------+ ++----------+------------+--------------+-------------+ +| | | TOTAL CHECKS | 0 | ++----------+------------+--------------+-------------+ +`), + }, + "sign-no-matching-severity": { + Writer: NewFakeWriter(""), + Sign: &internal.Signatures{ + Plugins: []internal.Plugin{ + { + Endpoints: []string{"/endpoint1", "/endpoint2"}, + Checks: []internal.Check{ + { + Name: "check-1", + Severity: "known-severity", + }, + }, + FollowRedirects: false, + }, + }, + }, + Sev: "unknown-severity", + ExpectedOutput: []byte(`+----------+------------+--------------+-------------+ +| ENDPOINT | CHECK NAME | SEVERITY | DESCRIPTION | ++----------+------------+--------------+-------------+ ++----------+------------+--------------+-------------+ +| | | TOTAL CHECKS | 0 | ++----------+------------+--------------+-------------+ +`), + }, + "sign-matching-severity": { + Writer: NewFakeWriter(""), + Sign: &internal.Signatures{ + Plugins: []internal.Plugin{ + { + Endpoints: []string{"/endpoint1", "/endpoint2"}, + Checks: []internal.Check{ + { + Name: "check-1", + Severity: "known-severity", + }, + }, + FollowRedirects: false, + }, + }, + }, + Sev: "known-severity", + ExpectedOutput: []byte(`+-------------------------+------------+----------------+-------------+ +| ENDPOINT | CHECK NAME | SEVERITY | DESCRIPTION | ++-------------------------+------------+----------------+-------------+ +| [/endpoint1 /endpoint2] | check-1 | known-severity | | ++-------------------------+------------+----------------+-------------+ +| | | TOTAL CHECKS | 1 | ++-------------------------+------------+----------------+-------------+ +`), + }, + } + + for testname, tt := range tests { + t.Run(testname, func(t *testing.T) { + internal.PrintSignatures(tt.Sign, tt.Sev, tt.Writer) + + if !reflect.DeepEqual(tt.Writer.Data(), tt.ExpectedOutput) { + t.Errorf("Failed to get expected output bytes: got \"%v\" instead of \"%v\".", tt.Writer.Data(), tt.ExpectedOutput) + } + }) + } +} diff --git a/main.go b/main.go deleted file mode 100644 index 633b639..0000000 --- a/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "gochopchop/cmd" - -func main() { - cmd.Execute() -} diff --git a/mock/formatting.go b/mock/formatting.go deleted file mode 100644 index f33e02b..0000000 --- a/mock/formatting.go +++ /dev/null @@ -1 +0,0 @@ -package mock diff --git a/mock/httpget.go b/mock/httpget.go deleted file mode 100644 index 0959b24..0000000 --- a/mock/httpget.go +++ /dev/null @@ -1,30 +0,0 @@ -package mock - -import ( - "bytes" - "fmt" - "gochopchop/internal/httpget" - "io/ioutil" - "net/http" -) - -type FakeNetClient map[string]*http.Response - -func (f FakeNetClient) Get(url string) (*http.Response, error) { - // implements IHTTPClient interface - if res, ok := f[url]; ok { - return res, nil - } - return nil, fmt.Errorf("could not get url : %s", url) -} - -var urls = FakeNetClient{ - "url1": &http.Response{ - StatusCode: 200, - Body: ioutil.NopCloser(bytes.NewReader([]byte("foo"))), - }, -} - -var FakeFetcher = &httpget.Fetcher{ - Netclient: urls, -} diff --git a/mock/output.go b/mock/output.go deleted file mode 100644 index dad0b7a..0000000 --- a/mock/output.go +++ /dev/null @@ -1,65 +0,0 @@ -package mock - -import ( - "gochopchop/core" -) - -var FakeOutputStatusCode = core.Output{ - URL: "http://problems", - Endpoint: FakePlugin.Endpoint, - Name: FakeCheckStatusCode200.Name, - Severity: FakeCheckStatusCode200.Severity, - Remediation: FakeCheckStatusCode200.Remediation, -} - -var FakeOutputMatchOne = core.Output{ - URL: "http://problems", - Endpoint: FakePlugin.Endpoint, - Name: FakeCheckMatchOne.Name, - Severity: FakeCheckMatchOne.Severity, - Remediation: FakeCheckMatchOne.Remediation, -} -var FakeOutputMatchAll = core.Output{ - URL: "http://problems", - Endpoint: FakePlugin.Endpoint, - Name: FakeCheckMatchAll.Name, - Severity: FakeCheckMatchAll.Severity, - Remediation: FakeCheckMatchAll.Remediation, -} - -var FakeOutputNotMatch = core.Output{ - URL: "http://problems", - Endpoint: FakePlugin.Endpoint, - Name: FakeCheckNotMatch.Name, - Severity: FakeCheckNotMatch.Severity, - Remediation: FakeCheckNotMatch.Remediation, -} - -var FakeOutputNoHeaders = core.Output{ - URL: "http://problems", - Endpoint: FakePlugin.Endpoint, - Name: FakeCheckNoHeaders.Name, - Severity: FakeCheckNoHeaders.Severity, - Remediation: FakeCheckNoHeaders.Remediation, -} - -var FakeOutputHeaders = core.Output{ - URL: "http://problems", - Endpoint: FakePlugin.Endpoint, - Name: FakeCheckHeaders.Name, - Severity: FakeCheckHeaders.Severity, - Remediation: FakeCheckHeaders.Remediation, -} - -var FakeOutput = []core.Output{ - FakeOutputStatusCode, - FakeOutputHeaders, - FakeOutputNoHeaders, - FakeOutputMatchAll, - FakeOutputMatchOne, - FakeOutputNotMatch, -} - -var FakeOutputAsCSV = "url,endpoint,severity,checkName,remediation\nhttp://problems,/,Medium,StatusCode200,uninstall\nhttp://problems,/,High,Headers,uninstall\nhttp://problems,/,Low,NoHeaders,uninstall\nhttp://problems,/,Informational,MustMatchAll,uninstall\nhttp://problems,/,Low,MustMatchOne,uninstall\nhttp://problems,/,High,MustNotMatch,uninstall\n" -var FakeOutputAsTable = "+-----------------+----------+---------------+---------------+-------------+\n| URL | ENDPOINT | SEVERITY | PLUGIN | REMEDIATION |\n+-----------------+----------+---------------+---------------+-------------+\n| http://problems | / | \x1b[31mHigh\x1b[0m | Headers | uninstall |\n| http://problems | / | \x1b[31mHigh\x1b[0m | MustNotMatch | uninstall |\n| http://problems | / | \x1b[32mLow\x1b[0m | NoHeaders | uninstall |\n| http://problems | / | \x1b[32mLow\x1b[0m | MustMatchOne | uninstall |\n| http://problems | / | \x1b[33mMedium\x1b[0m | StatusCode200 | uninstall |\n| http://problems | / | \x1b[36mInformational\x1b[0m | MustMatchAll | uninstall |\n+-----------------+----------+---------------+---------------+-------------+\n" -var FakeOutputAsJSON = "[{\"url\":\"http://problems\",\"endpoint\":\"/\",\"checkName\":\"StatusCode200\",\"severity\":\"Medium\",\"remediation\":\"uninstall\"},{\"url\":\"http://problems\",\"endpoint\":\"/\",\"checkName\":\"Headers\",\"severity\":\"High\",\"remediation\":\"uninstall\"},{\"url\":\"http://problems\",\"endpoint\":\"/\",\"checkName\":\"NoHeaders\",\"severity\":\"Low\",\"remediation\":\"uninstall\"},{\"url\":\"http://problems\",\"endpoint\":\"/\",\"checkName\":\"MustMatchAll\",\"severity\":\"Informational\",\"remediation\":\"uninstall\"},{\"url\":\"http://problems\",\"endpoint\":\"/\",\"checkName\":\"MustMatchOne\",\"severity\":\"Low\",\"remediation\":\"uninstall\"},{\"url\":\"http://problems\",\"endpoint\":\"/\",\"checkName\":\"MustNotMatch\",\"severity\":\"High\",\"remediation\":\"uninstall\"}]" diff --git a/mock/scan.go b/mock/scan.go deleted file mode 100644 index 7d19c37..0000000 --- a/mock/scan.go +++ /dev/null @@ -1,42 +0,0 @@ -package mock - -import ( - "fmt" - "gochopchop/core" - "gochopchop/internal" - "net/http" -) - -var FakeScanner = core.NewScanner(MyFakeFetcher, MyFakeFetcher, FakeSignatures, 1) - -type FakeFetcherWithoutNetclient map[string]*internal.HTTPResponse - -func (f FakeFetcherWithoutNetclient) Fetch(url string) (*internal.HTTPResponse, error) { - if res, ok := f[url]; ok { - return res, nil - } - return nil, fmt.Errorf("could not fetch : %s", url) -} - -var MyFakeFetcher = FakeFetcherWithoutNetclient{ - "http://problems/": &internal.HTTPResponse{ - StatusCode: 200, - Body: "MATCHONE lorem ipsum MATCHTWO", - Header: http.Header{ - "Header": []string{"ok"}, - "Header2": []string{"ok"}, - }, - }, - "http://noproblem/": &internal.HTTPResponse{ - StatusCode: 500, - Body: "NOTMATCH", - Header: http.Header{ - "Header": []string{"pasdutout"}, - "NoHeader": []string{"ok"}, - "NoHeader2": []string{"ok"}, - }, - }, - "http://noproblem/?query=test": &internal.HTTPResponse{ - StatusCode: 500, - }, -} diff --git a/mock/signatures.go b/mock/signatures.go deleted file mode 100644 index dfb07a6..0000000 --- a/mock/signatures.go +++ /dev/null @@ -1,122 +0,0 @@ -package mock - -import ( - "gochopchop/core" -) - -func createInt32(x int32) *int32 { - return &x -} - -// Checks - -var FakeCheckStatusCode200 = &core.Check{ - Name: "StatusCode200", - Severity: "Medium", - Remediation: "uninstall", - StatusCode: createInt32(200), -} - -var FakeCheckStatusCode500 = &core.Check{ - Name: "StatusCode500", - Severity: "High", - Remediation: "uninstall", - StatusCode: createInt32(500), -} - -var FakeCheckNoHeaders = &core.Check{ - Name: "NoHeaders", - Severity: "Low", - Remediation: "uninstall", - NoHeaders: []string{"NoHeader:ok"}, -} - -var FakeCheckNoHeadersKeyOnly = &core.Check{ - Name: "NoHeaders", - Severity: "Informational", - Remediation: "uninstall", - NoHeaders: []string{"NoHeader2"}, -} - -var FakeCheckHeaders = &core.Check{ - Name: "Headers", - Severity: "High", - Remediation: "uninstall", - Headers: []string{"Header:ok"}, -} -var FakeCheckHeaders2 = &core.Check{ - Name: "Headers", - Severity: "Medium", - Remediation: "uninstall", - Headers: []string{"Header2:ok"}, -} - -var FakeCheckMatchOne = &core.Check{ - Name: "MustMatchOne", - Severity: "Low", - Remediation: "uninstall", - MustMatchOne: []string{"MATCHONE", "MATCHTWO"}, -} - -var FakeCheckMatchAll = &core.Check{ - Name: "MustMatchAll", - Severity: "Informational", - Remediation: "uninstall", - MustMatchAll: []string{"MATCHONE", "MATCHTWO"}, -} - -var FakeCheckNotMatch = &core.Check{ - Name: "MustNotMatch", - Severity: "High", - Remediation: "uninstall", - MustNotMatch: []string{"NOTMATCH"}, -} - -// Plugins - -var FakePlugin = &core.Plugin{ - Endpoint: "/", - Checks: []*core.Check{ - FakeCheckStatusCode200, - FakeCheckHeaders, - FakeCheckHeaders2, - FakeCheckNoHeaders, - FakeCheckNoHeadersKeyOnly, - FakeCheckMatchAll, - FakeCheckMatchOne, - FakeCheckNotMatch, - }, -} - -var FakeQueryPlugin = &core.Plugin{ - Endpoint: "/", - QueryString: "query=test", - Checks: []*core.Check{ - FakeCheckStatusCode200, - }, -} - -var FakePlugin2 = &core.Plugin{ - Endpoint: "/fake", - QueryString: "query=test", - Checks: []*core.Check{ - FakeCheckStatusCode500, - }, -} - -var FakeFollowRedirectPlugin = &core.Plugin{ - Endpoint: "/", - Checks: []*core.Check{ - FakeCheckStatusCode200, - }, - FollowRedirects: true, -} - -// Signatures -var FakeSignatures = &core.Signatures{ - Plugins: []*core.Plugin{ - FakePlugin, - FakeQueryPlugin, - FakeFollowRedirectPlugin, - }, -} diff --git a/robot/data/common.resource b/robot/data/common.resource new file mode 100644 index 0000000..d6d49d9 --- /dev/null +++ b/robot/data/common.resource @@ -0,0 +1,44 @@ +*** Variables *** +${SIGNATURES_FILENAME}= chopchop.yml +${DS}= ${SPACE * 2} + +${SIGNATURES}= SEPARATOR=\n +... insecure: false +... plugins: +... ${DS}- endpoints: +... ${DS}${DS}${DS}- "/" +... ${DS}${DS}checks: +... ${DS}${DS}${DS}- name: EXAMPLE +... ${DS}${DS}${DS}${DS}match: +... ${DS}${DS}${DS}${DS}${DS}- 'Example' +... ${DS}${DS}${DS}${DS}remediation: Remediation +... ${DS}${DS}${DS}${DS}description: Description +... ${DS}${DS}${DS}${DS}severity: Informational +... ${DS}${DS}${DS}${DS}status_code: 200 + +${DOUBLE_SIGNATURES}= SEPARATOR=\n +... ${SIGNATURES} +... ${DS}- endpoints: +... ${DS}${DS}${DS}- "/2" +... ${DS}${DS}checks: +... ${DS}${DS}${DS}- name: EXAMPLE 2 +... ${DS}${DS}${DS}${DS}match: +... ${DS}${DS}${DS}${DS}${DS}- 'Example' +... ${DS}${DS}${DS}${DS}remediation: Remediation +... ${DS}${DS}${DS}${DS}description: Description +... ${DS}${DS}${DS}${DS}severity: Informational +... ${DS}${DS}${DS}${DS}status_code: 200 + +${FAILING_SIGNATURES}= SEPARATOR=\n +... insecure: false +... plugins: +... ${DS}- endpoints: +... ${DS}${DS}${DS}- "/" +... ${DS}${DS}checks: +... ${DS}${DS}${DS}- name: EXAMPLE +... ${DS}${DS}${DS}${DS}match: +... ${DS}${DS}${DS}${DS}${DS}- 'Example' +... ${DS}${DS}${DS}${DS}remediation: Remediation +... ${DS}${DS}${DS}${DS}description: Description +... ${DS}${DS}${DS}${DS}severity: Informational +... ${DS}${DS}${DS}${DS}status_code: 400 diff --git a/robot/libraries/ChopChop.py b/robot/libraries/ChopChop.py new file mode 100644 index 0000000..a92fbbb --- /dev/null +++ b/robot/libraries/ChopChop.py @@ -0,0 +1,39 @@ +import subprocess + +class ChopChop(object): + def __init__(self): + self.__stdout = '' + self.__returncode = None + + def chopchop_scan(self, signatures, url, threads='1'): + self.run_chopchop_command(['scan', '--signatures', signatures, '--export', 'stdout-no-color', '--threads', threads, url]) + + def chopchop_scan_url_file(self, signatures, url_file): + self.run_chopchop_command(['scan', '--signatures', signatures, '--export', 'stdout-no-color', '--url-file', url_file]) + + def chopchop_plugins(self, signatures): + self.run_chopchop_command(['plugins', '--signatures', signatures]) + + def run_chopchop_command(self, args): + args.insert(0, '../bin/chopchop') + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + + # Save and print Stdout + for line in proc.stdout: + ls = line.strip() + self.__stdout += ls + "\n" + print(ls) + + # Save and print return code + proc.communicate() + self.__returncode = proc.returncode + print('Return code', proc.returncode) + + def chopchop_get_stdout(self): + return self.__stdout + + def chopchop_get_returncode(self): + return self.__returncode diff --git a/robot/libraries/MockServer.py b/robot/libraries/MockServer.py new file mode 100644 index 0000000..0be9af7 --- /dev/null +++ b/robot/libraries/MockServer.py @@ -0,0 +1,39 @@ +import http.server +from threading import Thread, Barrier +from functools import partial + +class _WebMockServer(http.server.SimpleHTTPRequestHandler): + def log_message(self, format, *args): + pass # avoid logging + + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(bytes("Example", "utf8")) + +class MockServer(object): + def __init__(self): + self.__server = None + self.__thread = None + + def start_mock_server(self, port: int): + print("Starting server on port", port) + self.__server = http.server.HTTPServer(("", port), _WebMockServer) + + def serve_forever(server): + with server: + server.serve_forever() + + self.__thread = Thread(target=serve_forever, args=(self.__server, )) + self.__thread.setDaemon(True) + self.__thread.start() + + def stop_mock_server(self): + print("Stopping server") + if self.__server is not None: + self.__server.shutdown() + self.__thread.join() + + self.__server = None + self.__thread = None diff --git a/robot/run.sh b/robot/run.sh new file mode 100755 index 0000000..3b408ab --- /dev/null +++ b/robot/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +go build -o ../bin/chopchop ../cmd/main.go +robot --pythonpath libraries/ --outputdir out/ tests/ +rm -r ../bin diff --git a/robot/tests/plugins.robot b/robot/tests/plugins.robot new file mode 100644 index 0000000..3d45920 --- /dev/null +++ b/robot/tests/plugins.robot @@ -0,0 +1,34 @@ +*** Settings *** +Documentation A test suite to validate ChopChop CLI for the +... plugins command. + +Resource ../data/common.resource + +Library ChopChop +Library OperatingSystem + +*** Variables *** +${SIGNATURES_RESULTS}= SEPARATOR=\n +... +-----+-------------+---------------+-------------+ +... | URL | PLUGIN NAME | SEVERITY${DS}${DS}${DS}| DESCRIPTION | +... +-----+-------------+---------------+-------------+ +... | [/] | EXAMPLE${DS}${DS} | Informational | Description | +... +-----+-------------+---------------+-------------+ +... |${DS}${DS} |${DS}${DS}${DS}${DS}${DS}${DS} | TOTAL CHECKS${DS}| 1${DS}${DS}${DS}${DS}${DS} | +... +-----+-------------+---------------+-------------+ +... + +*** Test Cases *** +Simple Plugins + [Documentation] This test ensures ChopChop can display plugins. + [Tags] server + [Setup] Create File ${SIGNATURES_FILENAME} ${SIGNATURES} + [Teardown] Remove File ${SIGNATURES_FILENAME} + + Chopchop Plugins ${SIGNATURES_FILENAME} + + ${rc}= Chopchop Get Returncode + ${stdout}= Chopchop Get Stdout + + Should Be Equal As Integers ${rc} 0 + Should Be Equal As Strings ${stdout} ${SIGNATURES_RESULTS} diff --git a/robot/tests/scan.robot b/robot/tests/scan.robot new file mode 100644 index 0000000..38b9872 --- /dev/null +++ b/robot/tests/scan.robot @@ -0,0 +1,129 @@ +*** Settings *** +Documentation A test suite to validate ChopChop CLI for the +... scan command. + +Resource ../data/common.resource + +Library ChopChop +Library MockServer +Library OperatingSystem + +*** Variables *** +${PORT}= 8080 +${URL_FILE}= url-file.txt + +${SIGNATURES_RESULTS}= SEPARATOR=\n +... +-----------------------+----------+---------------+---------+-------------+ +... | URL${DS}${DS}${DS}${DS}${DS}${DS}${DS}${DS}${DS} | ENDPOINT | SEVERITY${DS}${DS}${DS}| PLUGIN${DS}| REMEDIATION | +... +-----------------------+----------+---------------+---------+-------------+ +... | http://127.0.0.1:${PORT} | /${DS}${DS}${DS}${DS}| Informational | EXAMPLE | Remediation | +... +-----------------------+----------+---------------+---------+-------------+ +... + +${DOUBLE_SIGNATURES_RESULTS}= SEPARATOR=\n +... +-----------------------+----------+---------------+-----------+-------------+ +... | URL${DS}${DS}${DS}${DS}${DS}${DS}${DS}${DS}${DS} | ENDPOINT | SEVERITY${DS}${DS}${DS}| PLUGIN${DS}${DS}| REMEDIATION | +... +-----------------------+----------+---------------+-----------+-------------+ +... | http://127.0.0.1:${PORT} | /${DS}${DS}${DS}${DS}| Informational | EXAMPLE${DS} | Remediation | +... | http://127.0.0.1:${PORT} | /2${DS}${DS}${DS} | Informational | EXAMPLE 2 | Remediation | +... +-----------------------+----------+---------------+-----------+-------------+ +... + +*** Test Cases *** +Simple Scan + [Documentation] This test ensures the basic job of ChopChop: scanning endpoints. + [Tags] server + [Setup] Setup Server And Signatures File ${PORT} ${SIGNATURES_FILENAME} ${SIGNATURES} + [Teardown] Teardown Server And Signatures File ${SIGNATURES_FILENAME} + + Chopchop Scan ${SIGNATURES_FILENAME} http://127.0.0.1:${PORT} + + ${rc}= Chopchop Get Returncode + ${stdout}= Chopchop Get Stdout + + Should Be Equal As Integers ${rc} 0 + Should Be Equal As Strings ${stdout} ${SIGNATURES_RESULTS} + +Scan With No Finding + [Documentation] This test ensures ChopChop fails in case there is no match. + [Tags] server + [Setup] Setup Server And Signatures File ${PORT} ${SIGNATURES_FILENAME} ${FAILING_SIGNATURES} + [Teardown] Teardown Server And Signatures File ${SIGNATURES_FILENAME} + + Chopchop Scan ${SIGNATURES_FILENAME} http://127.0.0.1:${PORT} + + ${rc}= Chopchop Get Returncode + + Should Be Equal As Integers ${rc} 1 + +Parallel Scan + [Documentation] This test ensures ChopChop works with multiple threads (or + ... concurrent routines). + [Tags] threads server + [Setup] Setup Server And Signatures File ${PORT} ${SIGNATURES_FILENAME} ${DOUBLE_SIGNATURES} + [Teardown] Teardown Server And Signatures File ${SIGNATURES_FILENAME} + + Chopchop Scan ${SIGNATURES_FILENAME} http://127.0.0.1:${PORT} 2 + + ${rc}= Chopchop Get Returncode + ${stdout}= Chopchop Get Stdout + + Should Be Equal As Integers ${rc} 0 + Should Be Equal As Strings ${stdout} ${DOUBLE_SIGNATURES_RESULTS} + +Scan With Url From File + [Documentation] This test ensures ChopChop works with url from file. + [Tags] server + [Setup] Setup Server Signatures And Url Files ${PORT} ${SIGNATURES_FILENAME} ${SIGNATURES} ${URL_FILE} http://127.0.0.1:${PORT} + [Teardown] Teardown Server Signatures And Url Files ${SIGNATURES_FILENAME} ${URL_FILE} + + Chopchop Scan Url File ${SIGNATURES_FILENAME} ${URL_FILE} + + ${rc}= Chopchop Get Returncode + ${stdout}= Chopchop Get Stdout + + Should Be Equal As Integers ${rc} 0 + Should Be Equal As Strings ${stdout} ${SIGNATURES_RESULTS} + +Invalid Signatures File + [Documentation] This test ensures it crashes if a given signatures file is + ... invalid. + [Tags] no-server + + Chopchop Scan ${SIGNATURES_FILENAME} http://127.0.0.1:${PORT} + + ${rc}= Chopchop Get Returncode + + Should Be Equal As Integers ${rc} 1 + +No URL provided + [Documentation] This test ensures it crashes if a given signatures file is + ... invalid. + [Tags] no-server + + Chopchop Scan ${SIGNATURES_FILENAME} / + + ${rc}= Chopchop Get Returncode + + Should Be Equal As Integers ${rc} 1 + +*** Keywords *** +Setup Server And Signatures File + [Arguments] ${port} ${sign_filename} ${sign} + Start Mock Server ${port} + Create File ${sign_filename} ${sign} + +Setup Server Signatures And Url Files + [Arguments] ${port} ${sign_filename} ${sign} ${url_filename} ${urls} + Setup Server And Signatures File ${port} ${sign_filename} ${sign} + Create File ${url_filename} ${urls} + +Teardown Server And Signatures File + [Arguments] ${sign_filename} + Remove File ${sign_filename} + Stop Mock Server + +Teardown Server Signatures And Url Files + [Arguments] ${sign_filename} ${url_filename} + Remove File ${url_filename} + Teardown Server And Signatures File ${sign_filename} diff --git a/tests/200.yml b/tests/200.yml deleted file mode 100644 index 79f6a8d..0000000 --- a/tests/200.yml +++ /dev/null @@ -1,738 +0,0 @@ -plugins: - - endpoint: "/" - checks: - - name: root 200 test - remediation: root - description: root - severity: "Medium" - status_code: 200 - - endpoint: "/status.shtml" - checks: - - name: GENEREX UPS - match: - - 'UPS Status:' - remediation: Make sure that GENEREX UPS access is restricted & monitored - description: GENEREX UPS is accessible | don't move this rule to avoid client timeout - severity: "Medium" - - endpoint: "/" - checks: - - name: Jenkins - match: - - "hudson" - remediation: Monitor access to jenkins only to trusted people, be sure that only connected people can run commands and that passwords are robust - description: Verifies that the domain is not a Jenkins instance - severity: "Informational" - headers: - - "Cache-Control:no-cache,no-store,must-revalidate" - - name : BigIPServer - remediation: Encrypt sticky cookie to avoid leaking internal IPs - description: Detects the presence of unencrypted sticky cookies that allow to retrieve internal Ips - severity: "Medium" - headers: - - "Set-Cookie:BIGipServer" - - name: TakeOver - match: - - "There is no app configured at that hostname" - - "NoSuchBucket" - - "No Such Account" - - "You're Almost There" - - "a GitHub Pages site here" - - "this shop is currently unavailable" - - "There's nothing here" - - "The site you were looking for couldn't be found" - - "The request could not be satisfied" - - "project not found" - - "Your CNAME settings" - - "The resource that you are attempting to access does not exist or you don't have the necessary permissions to view it." - - "Domain mapping upgrade for this domain not found" - - "The feed has not been found" - - "This UserVoice subdomain is currently available!" - remediation: Delete the DNS record as soon as possible - description: Detects the possibility of DNS Takeover - severity: High - - name: AsmxWebservices - match: - - ".asmx" - remediation: Monitor access to jenkins only to trusted people, be sure that only connected people can run commands and that passwords are robust - description: Verifies that the domain is not a Jenkins instance - severity: "Informational" - - name: Gitlab instance - match: - - "GitLab" - remediation: Make sure that access to Gitlab is properly monitored - description: Checks if a Gitlab instance exists - severity: "Low" - - name: Apache2 Ubuntu Default Page - match: - - "Apache2 Ubuntu Default Page" - remediation: Remove the symbolic link from the Apache default configuration - description: Detects the presence of a default Apache page - severity: "Informational" - - name: Drupal CMS - match: - - "drupal" - - '"sites/' - - '"core/' - remediation: Check that the version is the last one available on the vendor's website - description: Get the Drupal version of the site - severity: "Low" - - name: Status Code 500 - status_code: 500 - remediation: Check that the server has not completely fallen into error - description: Check return code 500 - severity: "Low" - - name: Iis - headers: - - "Server:Microsoft-IIS/6.0" - remediation: Patch the server as soon as possible - description: Verifies that the server is an IIS 6.0 - severity: "Informational" - - name: Indexof - match: - - "Index of" - remediation: Implementing rules at the application server level to prevent directory listing - description: Checks that the domain root does not return a file/folder list - severity: "Low" - - name: IndexOf2 - match: - - "<dir>" - remediation: Implementing rules at the application server level to prevent directory listing - description: Checks that the domain root does not return a file/folder list (simple encoding) - severity: "Low" - - name: MySQLError - match: - - 'You have an error in your SQL syntax' - remediation: Do not display MySQL errors on web pages - description: Checks that MySQL errors are not displayed - severity: "Medium" - - name: NginxDefaultPage - match: - - 'Welcome to nginx!' - remediation: Delete symbolic link from Nginx default configuration - description: Verifies that the default Nginx site is not accessible - severity: "Low" - - name: Osticket - match: - - 'Helpdesk software - powered by osTicket' - remediation: Check that the passwords used are robust - description: Verifies that the domain is not an OS Ticket instance - severity: "Informational" - - name: PHP open code - match: - - '' - remediation: Delete wildcards from xml files - description: Checks for the presence of a crossdomain.xml file with a wildcard for the domain - severity: "High" - - endpoint: "/manager/html" - checks: - - name: tomcat manager - status_code: 401 - remediation: Disable this interface in production - description: Verifies that under /manager/html the Tomcat administration interface is not accessible - severity: "Medium" - - endpoint: "/.htpasswd" - checks: - - name: .htpasswd not interpreted - match: - - ":" - remediation: Delete file and reset leaky passwords - description: Checks for the presence of an .htpasswd file at the root of the domain - severity: "Medium" - status_code: 200 - no_match: - - "' - remediation: Check that the administration interfaces are well protected - description: Detects the presence of a login page using the Apostrophe Framework (from Digital Factory) - severity: "Informational" - - name: Grafana - match: - - "isGrafanaAdmin" - remediation: Check that the passwords used are robust - description: Check access to Grafana administration - severity: "Informational" - - endpoint: "/user/login" - checks: - - name: eZ Publish Admin Panel - match: - - "Log in to the Administration Interface of eZ Publish" - remediation: Check that the passwords used are robust - description: Check access to the eZ Publish administration - severity: "Low" - - endpoint: "/fckeditor/editor/filemanager/browser/default/browser.html" - checks: - - name: FckEditor - match: - - "Resources Browser" - remediation: Put authentication on this form - description: Check access to a wysiwyg fckeditor - severity: "High" - - endpoint: "/.idea/workspace.xml" - checks: - - name: Idea WorkSpace - match: - - "phpMyAdmin' - remediation: Make sure that PHPMyAdmin access is monitored - description: Verifies that under /phpmyadmin a PHPMyAdmin instance is not accessible - status_code: 200 - severity: "Low" - - endpoint: "/server-status" - checks: - - name: Server Status - match: - - 'Waiting for Connection' - remediation: Disable this feature in Apache - description: Verifies that under /server-status of Apache information is accessible - severity: "Low" - - endpoint: "/examples/jsp/snp/snoop.jsp" - checks: - - name: Snoop - match: - - 'Request Information' - remediation: Delete basic files of a Tomcat installation - description: Detects the presence of snoop.jsp files (default files in a Tomcat install) - severity: "Informational" - - endpoint: "/actuator/health" - checks: - - name: SPringbootActuator - match: - - '{"status"' - headers: - - "Content-Type:application/json" - remediation: Disable this feature or protect access - description: Verifies that under /actuator/health information is not disclosed by Springboot - severity: "Low" - status_code: 200 - - endpoint: "/health" - checks: - - name: SpringbootActuator - match: - - '{"status"' - headers: - - "Content-Type:application/json" - remediation: Disable this feature or protect access - description: Verifies that under /health information is not disclosed by Springboot - severity: "Low" - status_code: 200 - - endpoint: "/.svn/wc.db" - checks: - - name: SVN db - headers: - - 'Content-Type:application/octet-stream' - remediation: Do not deploy .svn on production servers - description: Checks if an SVN database is publicly accessible - status_code : 200 - severity: "High" - - endpoint: "/.svn/entries" - checks: - - name: SVN db - headers: - - 'Content-Type:application/octet-stream' - remediation: Do not deploy .svn on production servers - description: Checks if an SVN database is publicly accessible - status_code: 200 - severity: "High" - - endpoint: "/web.config" - checks: - - name: Web Config - match: - - '' - remediation: Check that no sensitive information is present in the web.config - description: Verifies that the web.config configuration file of the ASP.net server is not accessible - severity: "Low" - - endpoint: "/wp-login.php" - checks: - - name: Wordpress Login Page - all_match: - - 'wp-login.php" method="post"' - - 'BIG-IP" - remediation: Make sure that F5 BIG-IP - TMUI access is monitored - description: Verifies that under /tmui a F5 BIG-IP TMUI is not accessible - severity: "Low" - - endpoint: "/images/imgpaper.png" - checks: - - name : Possible Trickbot Trojan Payload hosting imgpaper.png on Apache - headers: - - 'Content-Type:image/png' - remediation: Make sure your system isn't compromised - description: Possible Trickbot Trojan Payload hosting in /images/imgpaper.png - status_code: 200 - severity: "High" - - endpoint: "/images/imgpaper.png" - checks: - - name : Trickbot Trojan Payload hosting imgpaper.png on Nginx - headers: - - 'Content-Type:application/octet-stream' - remediation: Make sure your system isn't compromised - description: Possible Trickbot Trojan Payload hosting in /images/imgpaper.png - status_code: 200 - severity: "High" - - endpoint: "/images/cursor.png" - checks: - - name: Possible Trickbot Trojan Payload hosting cursor.png on Apache - headers: - - 'Content-Type:image/png' - remediation: Make sure your system isn't compromised - description: Possible Trickbot Trojan Payload hosting in /images/cursor.png - status_code: 200 - severity: "High" - - endpoint: "/images/cursor.png" - checks: - - name: Trickbot Trojan Payload hosting cursor.png on Nginx - headers: - - 'Content-Type:application/octet-stream' - remediation: Make sure your system isn't compromised - description: Possible Trickbot Trojan Payload hosting in /images/cursor.png - status_code: 200 - severity: "High" - - endpoint: "/images/redcar.png" - checks: - - name : Possible Trickbot Trojan Payload hosting redcar.png on Apache - headers: - - 'Content-Type:image/png' - remediation: Make sure your system isn't compromised - description: Possible Trickbot Trojan Payload hosting in /images/redcar.png - status_code: 200 - severity: "High" - - endpoint: "/images/redcar.png" - checks: - - name : Trickbot Trojan Payload hosting redcar.png on Nginx - headers: - - 'Content-Type:application/octet-stream' - remediation: Make sure your system isn't compromised - description: Possible Trickbot Trojan Payload hosting in /images/redcar.png - status_code: 200 - severity: "High" - - endpoint: "/ico/VidT6cErs" - checks: - - name : Possible Trickbot Trojan Payload hosting VidT6cErs - no_match: - - '' - - '' - headers: - - "Accept-Ranges:bytes" - remediation: Make sure your system isn't compromised - description: Possible Trickbot Trojan Payload hosting in /ico/VidT6cErs - status_code: 200 - severity: "High" - - endpoint: "/admin/libs/prettify-4-Mar-2013/prettify.css" - checks: - - name : Stormshield SNS Web Admin Console - headers: - - "Content-Type:text/css" - remediation: Make sure that Stormshield SNS Web Admin Console access is restricted & monitored - description: Stormshield SNS Web Admin Console is accessible - status_code: 200 - severity: "Low" - - endpoint: "/auth" - checks: - - name : Stormshield Web Portal - match: - - '/data/flag-fr.jpg' - - '/data/i_auth.png' - remediation: Make sure that Stormshield Web Portal access is restricted & monitored - description: Stormshield Web Portal is accessible - status_code: 200 - severity: "Informational" - - endpoint: "/restgui/start.html" - checks: - - name : Dell IDRAC - headers: - - "Content-Type:text/html" - remediation: Make sure that Dell IDRAC access is restricted & monitored - description: Dell IDRAC is accessible - status_code: 200 - severity: "Informational" - - endpoint: "/ui" - checks: - - name : VMware ESXi - match: - - 'ng-app="esxUiApp"' - remediation: Make sure that VMware ESXi access is restricted & monitored - description: VMware ESXi is accessible - status_code: 200 - severity: "Low" - - endpoint: "/vsphere-client" - checks: - - name : VMware vCenter - match: - - 'vSphere Web Client' - remediation: Make sure that VMware vCenter access is restricted & monitored - description: VMware vCenter is accessible - status_code: 200 - severity: "Low" - - endpoint: "/eai/index.html" - checks: - - name : Enovacom Suite V2 - match: - - 'href="/eai/Ressources/Images/v2.ico"' - remediation: Make sure that EAI Enovacom Suite V2 access is restricted & monitored - description: EAI Enovacom Suite V2 is accessible - status_code: 200 - severity: "Low" - - endpoint: "/mailscanner/login.php" - checks: - - name : MailWatch - match: - - 'MailWatch Login Page' - remediation: Make sure that MailWatch access is monitored - description: MailWatch is accessible - status_code: 200 - severity: "Low" - - endpoint: "/fog/management/index.php" - checks: - - name : FOG Project - match: - - 'Login' - - 'FOG Project' - remediation: Make sure that FOG Project access is monitored - description: FOG Project is accessible - status_code: 200 - severity: "Low" - - endpoint: "/.well-known/security.txt" - checks: - - name: Security.txt - match: - - "Contact" - remediation: Great ! A Security.txt file for contact is present - description: Detects the presence of Security.txt file - status_code: 200 - severity: "Informational" - - endpoint: "/" - checks: - - name : Microsoft-IIS/7.0 - Windows Server 2003/2008 - headers: - - "Server:Microsoft-IIS/7.0" - remediation: Upgrade to maintened version - description: Microsoft-IIS/7.0 - Windows Server 2003/2008 - severity: "Informational" - - name : Microsoft-IIS/7.5 - Windows Server 2003/2008 - headers: - - "Server:Microsoft-IIS/7.5" - remediation: Upgrade to maintened version - description: Microsoft-IIS/7.5 - Windows Server 2003/2008 - severity: "Informational" - - endpoint: "/" - checks: - - name : GE ViewPoint - match: - - 'ViewPoint System Status' - remediation: Make sure that GE ViewPoint System Status access is restricted & monitored - description: GE ViewPoint System Status is accessible / sensitive information leaking - status_code: 200 - severity: "Low" - - endpoint: "/" - checks: - - name : Ascom IP-DECT Base Station - match: - - '<select product="Ascom IP-DECT Base Station"' - remediation: Make sure that Ascom IP-DECT Base Station access is restricted & monitored - description: Ascom IP-DECT Base Station is accessible - status_code: 200 - severity: "Informational" - - endpoint: "/" - checks: - - name : EMC Unisphere - match: - - 'Unisphere<br>' - remediation: Make sure that EMC Unisphere access is restricted & monitored - description: EMC Unisphere is accessible - status_code: 200 - severity: "Low" - - endpoint: "/" - checks: - - name : F-Secure Policy Manager Server - match: - - '<title>F-Secure Policy Manager Server' - remediation: Make sure that F-Secure Policy Manager Server access is monitored - description: F-Secure Policy Manager Server is accessible - status_code: 200 - severity: "Informational" - - endpoint: "/" - checks: - - name: Apache2 Debian Default Page - match: - - 'Apache2 Debian Default Page: It works' - remediation: Remove the symbolic link from the Apache default configuration - description: Detects the presence of a default Apache page - severity: "Informational" - - endpoint: "/" - checks: - - name : Cisco IOS - headers: - - "Server:cisco-IOS" - remediation: Make sure that Cisco IOS access is restricted & monitored - description: Cisco IOS is accessible - severity: "Low" - - endpoint: "/XsEXPL" - checks: - - name: Xplore Web RIS - match: - - 'Xplore Exploitation' - remediation: Make sure that Xplore Web RIS access is restricted & monitored - description: Xplore Web RIS is accessible - severity: "Informational" - - endpoint: "/zimbraAdmin" - checks: - - name: Zimbra Administration - match: - - 'Zimbra Collaboration Suite Web Client' - remediation: Make sure that Zimbra Administration access is restricted & monitored - description: Zimbra Administration is accessible - severity: "Low" - - endpoint: "/" - checks: - - name: NETAVIS Observer - match: - - 'NETAVIS Observer' - remediation: Make sure that NETAVIS Observer access is restricted & monitored - description: NETAVIS Observer is accessible - severity: "Informational" - - endpoint: "/" - checks: - - name : Odin - match: - - '<h1 title="Operations Automation">' - remediation: Make sure that Odin service automation access is restricted & monitored - description: Odin service automation is accessible - severity: "Informational" - - endpoint: "/" - checks: - - name : Nordex Control - headers: - - "Server:Jetty/3.1.8 (Windows 2000 5.0 x86)" - remediation: Make sure that Nordex Control access is restricted & monitored - description: Nordex Control is accessible - severity: "Low" - - endpoint: "/" - checks: - - name : EIG GaugeTech Electricity Meter - headers: - - "Server:EIG Embedded Web Server" - remediation: Make sure that EIG GaugeTech Electricity Meter access is restricted & monitored - description: EIG GaugeTech Electricity Meter is accessible - severity: "Low" - - endpoint: "/" - checks: - - name : Weave Scope - match: - - '<title>Weave Scope' - remediation: Make sure that Weave Scope access is restricted & monitored - description: Weave Scope is accessible - severity: "Medium" - - endpoint: "/public/img/mongo-express-logo.png" - checks: - - name : Mongo Express - headers: - - 'Content-Type:image/png' - remediation: Make sure that Mongo Express access is restricted & monitored - description: Mongo Express is accessible - status_code: 200 - severity: "High" - - endpoint: "/login.html" - checks: - - name : Polycom - headers: - - 'Server:lighttpd' - match: - - 'Polycom Login' - remediation: Make sure that Polycom access is restricted & monitored - description: Polycom Video Conferencing is accessible - status_code: 200 - severity: "Informational" - - endpoint: "/securityRealm/user/admin/search/index?q=a" - checks: - - name : Jenkins CVE-2018-1000861 (RCE) - match: - - 'Jenkins' - - 'Search for' - no_match: - - 'HTTP ERROR 404 Not Found' - remediation: Patch the server as soon as possible - description: Jenkins server is vulnerable to RCE CVE-2018-1000861 - severity: "High" - - endpoint: "/?MAIN=TOPACCESS" - checks: - - name : TopAccess Toshiba MFP - match: - - '<!--<title class="clsTitle1">TopAccess-->' - remediation: Make sure that TopAccess access is restricted & monitored - description: TopAccess Toshiba MFP is accessible - status_code: 200 - severity: "Low" - - endpoint: "/" - checks: - - name : HP Printer - headers: - - "Server:Virata-EmWeb/R6_2_1" - remediation: Make sure that HP Printer access is restricted & monitored - description: HP Printer is accessible - status_code: 200 - severity: "Low" - - endpoint: "/ePrint/ePrintConfigDyn.xml" - checks: - - name : HP Printer - headers: - - 'Content-Type:text/xml' - remediation: Make sure that HP Printer access is restricted & monitored - description: HP Printer is accessible - status_code: 200 - severity: "Low" - - endpoint: "/" - checks: - - name : Printer (Lexmark, Dell, Toshiba, Sindoh) - headers: - - "Server:Lexmark_Web_Server" - remediation: Make sure that Printer access is restricted & monitored - description: Printer (Lexmark, Dell, Toshiba, Sindoh) is accessible - status_code: 200 - severity: "Low" - - endpoint: "/config.html" - checks: - - name : Zebra Label Printer - match: - - '

Zebra Technologies' - remediation: Make sure that Zebra Label Printer access is restricted & monitored - description: Zebra Label Printer is accessible - status_code: 200 - severity: "Low" diff --git a/tests/server.go b/tests/server.go deleted file mode 100644 index 5a777fb..0000000 --- a/tests/server.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "fmt" - "net/http" -) - -func main() { - http.HandleFunc("/", HelloServer) - http.ListenAndServe(":8000", nil) -} - -func HelloServer(w http.ResponseWriter, r *http.Request) { - fmt.Println("hit") - fmt.Fprintf(w, "Hello, man!") -}