From 534e1f01442cafaa932e8952ee476000d20f11f0 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzhal from RunPod Date: Tue, 14 Nov 2023 11:30:16 +0200 Subject: [PATCH 01/14] Dockerize initial commit --- .github/workflows/generate-image.yml | 66 +++++++++++++++++ Dockerfile | 28 +++++++ Dockerfile-basic | 24 ++++++ Dockerfile-update-nvd | 5 ++ Dockerfile-update-nvd2osv | 4 + build-all.sh | 4 + config.toml | 5 +- db/create.sql | 6 ++ pg_hba.conf | 105 +++++++++++++++++++++++++++ vulndb.sh | 7 ++ 10 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/generate-image.yml create mode 100644 Dockerfile create mode 100644 Dockerfile-basic create mode 100644 Dockerfile-update-nvd create mode 100644 Dockerfile-update-nvd2osv create mode 100644 build-all.sh create mode 100644 pg_hba.conf create mode 100755 vulndb.sh diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml new file mode 100644 index 00000000..fd414789 --- /dev/null +++ b/.github/workflows/generate-image.yml @@ -0,0 +1,66 @@ +# +name: Create and publish a Docker image + +# Configures this workflow to run every time a change is pushed to the branch called `release`. +on: + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build&Push Basic Image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: vulndb:basic + labels: ${{ steps.meta.outputs.labels }} + file: Dockerfile-basic + + - name: Build&Push Image with updated nvd data + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: vulndb:nvd + labels: ${{ steps.meta.outputs.labels }} + file: Dockerfile-update-nvd + + - name: Build&Push Image with updated nvd + osv + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: vulndb:nvd-osv, vunldb:latest + labels: ${{ steps.meta.outputs.labels }} + file: Dockerfile-basic diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..118dddec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.10 +ARG PACKAGE +COPY db/create.sql /docker-entrypoint-initdb.d/. +RUN apt-get update && apt-get install -y postgresql postgresql-contrib +RUN apt-get install -y gcc +ENV DB_USER=default_user +ENV DB_PASSWORD=default_password +ENV DB_NAME=default_db +USER postgres +RUN /etc/init.d/postgresql start && \ + psql --command "CREATE USER ${DB_USER} WITH SUPERUSER PASSWORD '${DB_PASSWORD}';" +USER root +RUN pip install pip +RUN pip install psycopg2 +WORKDIR /app +COPY . /app +COPY config.toml /app/src/bomsquad/vulndb/config.toml +COPY ${PACKAGE} /app +COPY pg_hba.conf /etc/postgresql/15/main/pg_hba.conf +RUN pip install -e . +USER postgres +RUN /etc/init.d/postgresql start && psql < /app/db/create.sql +RUN /etc/init.d/postgresql start && vulndb nvd ingest +RUN /etc/init.d/postgresql start && vulndb osv ingest +EXPOSE 5000 +ENTRYPOINT [ "/app/vulndb.sh" ] + + diff --git a/Dockerfile-basic b/Dockerfile-basic new file mode 100644 index 00000000..820e21ba --- /dev/null +++ b/Dockerfile-basic @@ -0,0 +1,24 @@ +FROM python:3.10 +ARG PACKAGE +COPY db/create.sql /docker-entrypoint-initdb.d/. +RUN apt-get update && apt-get install -y postgresql postgresql-contrib +RUN apt-get install -y gcc +ENV DB_USER=default_user +ENV DB_PASSWORD=default_password +ENV DB_NAME=default_db +USER postgres +RUN /etc/init.d/postgresql start && \ + psql --command "CREATE USER ${DB_USER} WITH SUPERUSER PASSWORD '${DB_PASSWORD}';" +USER root +RUN pip install pip +RUN pip install psycopg2 +WORKDIR /app +COPY . /app +COPY config.toml /app/src/bomsquad/vulndb/config.toml +COPY ${PACKAGE} /app +COPY pg_hba.conf /etc/postgresql/15/main/pg_hba.conf +RUN pip install -e . +USER postgres +RUN /etc/init.d/postgresql start && psql < /app/db/create.sql +EXPOSE 5000 +ENTRYPOINT [ "/app/vulndb.sh" ] \ No newline at end of file diff --git a/Dockerfile-update-nvd b/Dockerfile-update-nvd new file mode 100644 index 00000000..25384448 --- /dev/null +++ b/Dockerfile-update-nvd @@ -0,0 +1,5 @@ +FROM vulndb:basic +RUN /etc/init.d/postgresql start && vulndb nvd ingest --scope cve --update +#RUN /etc/init.d/postgresql start && vulndb osv ingest +EXPOSE 5000 +ENTRYPOINT [ "/app/vulndb.sh" ] \ No newline at end of file diff --git a/Dockerfile-update-nvd2osv b/Dockerfile-update-nvd2osv new file mode 100644 index 00000000..6ec1b4ff --- /dev/null +++ b/Dockerfile-update-nvd2osv @@ -0,0 +1,4 @@ +FROM vulndb:nvd +RUN /etc/init.d/postgresql start && vulndb osv ingest +EXPOSE 5000 +ENTRYPOINT [ "/app/vulndb.sh" ] \ No newline at end of file diff --git a/build-all.sh b/build-all.sh new file mode 100644 index 00000000..71eeb889 --- /dev/null +++ b/build-all.sh @@ -0,0 +1,4 @@ +docker build -t vulndb:basic -f Dockerfile-basic . +docker build -t vulndb:nvd -f Dockerfile-update-nvd . +docker build -t vulndb:nvd-osv -f Dockerfile-update-nvd2osv . +docker tag vulndb:nvd-osv vulndb diff --git a/config.toml b/config.toml index 3c5c099c..fec7e2c3 100644 --- a/config.toml +++ b/config.toml @@ -1,8 +1,11 @@ [vulndb] -database = 'vulndb' +database='vulndb' min_conn = 1 max_conn = 10 +username='default_user' +password='default_password' # username = 'your-username' # password = 'your-password' # nvd_api_key = 'your-api-key' request_delay = 5 +gcloud_project = 'devstartup' diff --git a/db/create.sql b/db/create.sql index f1b81d13..6a3586c6 100644 --- a/db/create.sql +++ b/db/create.sql @@ -1,5 +1,7 @@ CREATE DATABASE vulndb; +\c vulndb; + CREATE TABLE cve( id serial NOT NULL PRIMARY KEY, data jsonb NOT NULL @@ -22,3 +24,7 @@ CREATE TABLE osv( CREATE INDEX osv_id ON osv USING gin((data->'id')); CREATE INDEX osv_alias_id ON osv USING gin((data->'aliases')); + +GRANT CONNECT ON DATABASE vulndb TO default_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO default_user; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO default_user; diff --git a/pg_hba.conf b/pg_hba.conf new file mode 100644 index 00000000..d4679b6f --- /dev/null +++ b/pg_hba.conf @@ -0,0 +1,105 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: +# - "local" is a Unix-domain socket +# - "host" is a TCP/IP socket (encrypted or not) +# - "hostssl" is a TCP/IP socket that is SSL-encrypted +# - "hostnossl" is a TCP/IP socket that is not SSL-encrypted +# - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted +# - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the server receives a +# SIGHUP signal. If you edit the file on a running system, you have to +# SIGHUP the server for the changes to take effect, run "pg_ctl reload", +# or execute "SELECT pg_reload_conf()". +# +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + + + + +# DO NOT DISABLE! +# If you change this first entry you will need to make sure that the +# database superuser can access the database using some other method. +# Noninteractive access to all databases is required during automatic +# maintenance (custom daily cronjobs, replication, and similar tasks). +# +# Database administrative login by Unix domain socket +local all postgres peer + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +host all all 127.0.0.1/32 scram-sha-256 +# IPv6 local connections: +host all all ::1/128 scram-sha-256 +# Allow replication connections from localhost, by a user with the +# replication privilege. +local replication all peer +host replication all 127.0.0.1/32 scram-sha-256 +host replication all ::1/128 scram-sha-256 + diff --git a/vulndb.sh b/vulndb.sh new file mode 100755 index 00000000..28d541b6 --- /dev/null +++ b/vulndb.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# This script forwards all arguments to the vulndb command + +# Forwarding arguments +/etc/init.d/postgresql start +vulndb "$@" From 4978272341e43a37e6f72efd1427c706ba6e7c46 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 11:50:47 +0200 Subject: [PATCH 02/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index fd414789..a609dca2 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -43,7 +43,7 @@ jobs: with: context: . push: true - tags: vulndb:basic + tags: basic labels: ${{ steps.meta.outputs.labels }} file: Dockerfile-basic From eb227ba2192bf66522e5ef23c21823d480a1e623 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 12:02:42 +0200 Subject: [PATCH 03/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index a609dca2..f182f9a5 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -28,7 +28,7 @@ jobs: with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GH_TOKEN }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta @@ -47,20 +47,20 @@ jobs: labels: ${{ steps.meta.outputs.labels }} file: Dockerfile-basic - - name: Build&Push Image with updated nvd data - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - push: true - tags: vulndb:nvd - labels: ${{ steps.meta.outputs.labels }} - file: Dockerfile-update-nvd + # - name: Build&Push Image with updated nvd data + # uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + # with: + # context: . + # push: true + # tags: vulndb:nvd + # labels: ${{ steps.meta.outputs.labels }} + # file: Dockerfile-update-nvd - - name: Build&Push Image with updated nvd + osv - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - push: true - tags: vulndb:nvd-osv, vunldb:latest - labels: ${{ steps.meta.outputs.labels }} - file: Dockerfile-basic + # - name: Build&Push Image with updated nvd + osv + # uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + # with: + # context: . + # push: true + # tags: vulndb:nvd-osv, vunldb:latest + # labels: ${{ steps.meta.outputs.labels }} + # file: Dockerfile-basic From 531683fa39721c75949cf4e1bd4ffe0a04fe71b0 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 12:09:14 +0200 Subject: [PATCH 04/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index f182f9a5..0c3d1edd 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -24,7 +24,7 @@ jobs: uses: actions/checkout@v4 # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -39,7 +39,7 @@ jobs: # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - name: Build&Push Basic Image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + uses: docker/build-push-action@v5 with: context: . push: true From ebc33d05ea3810defae4310209ccd4fc9dbe7044 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 12:39:03 +0200 Subject: [PATCH 05/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index 0c3d1edd..91bed173 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -15,10 +15,10 @@ jobs: build-and-push-image: runs-on: ubuntu-latest # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - # + permissions: write-all + # contents: read-all + # packages: write-all + # # steps: - name: Checkout repository uses: actions/checkout@v4 @@ -28,7 +28,7 @@ jobs: with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} - password: ${{ secrets.GH_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta From 4f392f4928bf03e5a944d8271607a1eea21133ed Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 12:45:38 +0200 Subject: [PATCH 06/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index 91bed173..0442fc31 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -7,7 +7,7 @@ on: # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: - REGISTRY: ghcr.io + #REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. @@ -15,10 +15,10 @@ jobs: build-and-push-image: runs-on: ubuntu-latest # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: write-all - # contents: read-all - # packages: write-all - # # + # permissions: write-all + # # contents: read-all + # # packages: write-all + # # # steps: - name: Checkout repository uses: actions/checkout@v4 @@ -27,8 +27,8 @@ jobs: uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta @@ -43,7 +43,7 @@ jobs: with: context: . push: true - tags: basic + tags: scribesecurity/vulndb:basic labels: ${{ steps.meta.outputs.labels }} file: Dockerfile-basic From ee12807c34804d4eecbe6a7c0a100b45265e5d5a Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 12:56:05 +0200 Subject: [PATCH 07/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 58 +++++++++------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index 0442fc31..a41d8bc6 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -4,63 +4,41 @@ name: Create and publish a Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: workflow_dispatch: - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - #REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. + jobs: build-and-push-image: runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - # permissions: write-all - # # contents: read-all - # # packages: write-all - # # # steps: - name: Checkout repository uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build&Push Basic Image uses: docker/build-push-action@v5 with: context: . push: true - tags: scribesecurity/vulndb:basic - labels: ${{ steps.meta.outputs.labels }} + tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:basic file: Dockerfile-basic - # - name: Build&Push Image with updated nvd data - # uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - # with: - # context: . - # push: true - # tags: vulndb:nvd - # labels: ${{ steps.meta.outputs.labels }} - # file: Dockerfile-update-nvd + - name: Build&Push Image with updated nvd data + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:nvd + file: Dockerfile-update-nvd - # - name: Build&Push Image with updated nvd + osv - # uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - # with: - # context: . - # push: true - # tags: vulndb:nvd-osv, vunldb:latest - # labels: ${{ steps.meta.outputs.labels }} - # file: Dockerfile-basic + - name: Build&Push Image with updated nvd + osv + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:nvd-osv, ${{secrets.DOCKERHUB_USERNAME}}/vunldb:latest + file: Dockerfile-basic From 099fae02d21a1984c24c0b7d9d12cf0ca8ae702f Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 13:03:54 +0200 Subject: [PATCH 08/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index a41d8bc6..34e42433 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -18,6 +18,11 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{secrets.DOCKERHUB_USERNAME}}/vulndb - name: Build&Push Basic Image uses: docker/build-push-action@v5 @@ -25,6 +30,7 @@ jobs: context: . push: true tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:basic + labels: ${{ steps.meta.outputs.labels }} file: Dockerfile-basic - name: Build&Push Image with updated nvd data @@ -33,6 +39,7 @@ jobs: context: . push: true tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:nvd + labels: ${{ steps.meta.outputs.labels }} file: Dockerfile-update-nvd - name: Build&Push Image with updated nvd + osv @@ -41,4 +48,5 @@ jobs: context: . push: true tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:nvd-osv, ${{secrets.DOCKERHUB_USERNAME}}/vunldb:latest + labels: ${{ steps.meta.outputs.labels }} file: Dockerfile-basic From 885f9cb0f731970ac5957fd876822e7b1cb82587 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzhal from RunPod Date: Tue, 14 Nov 2023 14:11:28 +0200 Subject: [PATCH 09/14] fixes for handling FROM --- .github/workflows/generate-image.yml | 18 +++++-------- .github/workflows/update-image.yml | 39 ++++++++++++++++++++++++++++ Dockerfile-update | 5 ++++ 3 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/update-image.yml create mode 100644 Dockerfile-update diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index 34e42433..3e2a34c1 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -33,20 +33,16 @@ jobs: labels: ${{ steps.meta.outputs.labels }} file: Dockerfile-basic - - name: Build&Push Image with updated nvd data - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:nvd - labels: ${{ steps.meta.outputs.labels }} - file: Dockerfile-update-nvd - - name: Build&Push Image with updated nvd + osv + - name: Update FROM in update Docker file + run: sed -i 's|FROM vulndb:latest|FROM ${{DOCKERHUB_USERNAME}}/vulndb:basic|' Dockerfile-update + + + - name: Build&Push Image with updated data uses: docker/build-push-action@v5 with: context: . push: true - tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:nvd-osv, ${{secrets.DOCKERHUB_USERNAME}}/vunldb:latest + tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:latest labels: ${{ steps.meta.outputs.labels }} - file: Dockerfile-basic + file: Dockerfile-update diff --git a/.github/workflows/update-image.yml b/.github/workflows/update-image.yml new file mode 100644 index 00000000..4455fade --- /dev/null +++ b/.github/workflows/update-image.yml @@ -0,0 +1,39 @@ +# +name: Create and publish a Docker image + +# Configures this workflow to run every time a change is pushed to the branch called `release`. +on: + workflow_dispatch: + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{secrets.DOCKERHUB_USERNAME}}/vulndb + + - name: Update FROM in update Docker file + run: sed -i 's|FROM vulndb:latest|FROM ${{DOCKERHUB_USERNAME}}/vulndb:latest|' Dockerfile-update + + + - name: Build&Push Image with updated data + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{secrets.DOCKERHUB_USERNAME}}/vulndb:latest + labels: ${{ steps.meta.outputs.labels }} + file: Dockerfile-update diff --git a/Dockerfile-update b/Dockerfile-update new file mode 100644 index 00000000..cd5d6892 --- /dev/null +++ b/Dockerfile-update @@ -0,0 +1,5 @@ +FROM vulndb:latest +RUN /etc/init.d/postgresql start && vulndb nvd ingest --scope cve --update +RUN /etc/init.d/postgresql start && vulndb osv ingest --update +EXPOSE 5000 +ENTRYPOINT [ "/app/vulndb.sh" ] \ No newline at end of file From 03d94fd3de43fa4558fd8107c326ca2a5530efe4 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 14:12:39 +0200 Subject: [PATCH 10/14] Update update-image.yml --- .github/workflows/update-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-image.yml b/.github/workflows/update-image.yml index 4455fade..d8fb87e4 100644 --- a/.github/workflows/update-image.yml +++ b/.github/workflows/update-image.yml @@ -1,5 +1,5 @@ # -name: Create and publish a Docker image +name: Update and publish vunldb Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: From bbbf11d6c20b6759b9c9ab98e3e74de42947dc31 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 14:14:43 +0200 Subject: [PATCH 11/14] Update generate-image.yml --- .github/workflows/generate-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-image.yml b/.github/workflows/generate-image.yml index 3e2a34c1..bdc325ad 100644 --- a/.github/workflows/generate-image.yml +++ b/.github/workflows/generate-image.yml @@ -35,7 +35,7 @@ jobs: - name: Update FROM in update Docker file - run: sed -i 's|FROM vulndb:latest|FROM ${{DOCKERHUB_USERNAME}}/vulndb:basic|' Dockerfile-update + run: sed -i 's|FROM vulndb:latest|FROM ${{secrets.DOCKERHUB_USERNAME}}/vulndb:basic|' Dockerfile-update - name: Build&Push Image with updated data From befab64eb9a86b802adfec9ecb18bc3175df6ce4 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 14:15:19 +0200 Subject: [PATCH 12/14] Update update-image.yml --- .github/workflows/update-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-image.yml b/.github/workflows/update-image.yml index d8fb87e4..800b87d3 100644 --- a/.github/workflows/update-image.yml +++ b/.github/workflows/update-image.yml @@ -26,7 +26,7 @@ jobs: images: ${{secrets.DOCKERHUB_USERNAME}}/vulndb - name: Update FROM in update Docker file - run: sed -i 's|FROM vulndb:latest|FROM ${{DOCKERHUB_USERNAME}}/vulndb:latest|' Dockerfile-update + run: sed -i 's|FROM vulndb:latest|FROM ${{secrets.DOCKERHUB_USERNAME}}/vulndb:latest|' Dockerfile-update - name: Build&Push Image with updated data From 8e187d46b8d65b25ae4460d2d5c0e38d25be54e2 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzahl Date: Tue, 14 Nov 2023 14:41:34 +0200 Subject: [PATCH 13/14] Update Dockerfile-update --- Dockerfile-update | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile-update b/Dockerfile-update index cd5d6892..0d9e1a16 100644 --- a/Dockerfile-update +++ b/Dockerfile-update @@ -1,5 +1,5 @@ FROM vulndb:latest RUN /etc/init.d/postgresql start && vulndb nvd ingest --scope cve --update -RUN /etc/init.d/postgresql start && vulndb osv ingest --update +RUN /etc/init.d/postgresql start && vulndb osv ingest EXPOSE 5000 -ENTRYPOINT [ "/app/vulndb.sh" ] \ No newline at end of file +ENTRYPOINT [ "/app/vulndb.sh" ] From 9d346a86a36321f19552c7499c282646e00fb5f2 Mon Sep 17 00:00:00 2001 From: Daniel Nebenzhal from RunPod Date: Sun, 3 Dec 2023 09:44:59 +0200 Subject: [PATCH 14/14] fixes to project definition file to include requests + fix docker to non include cpe --- Dockerfile | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 118dddec..10a7a437 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.10 ARG PACKAGE COPY db/create.sql /docker-entrypoint-initdb.d/. -RUN apt-get update && apt-get install -y postgresql postgresql-contrib +RUN apt-get clean && apt-get update && apt-get install -y postgresql postgresql-contrib RUN apt-get install -y gcc ENV DB_USER=default_user ENV DB_PASSWORD=default_password @@ -20,7 +20,7 @@ COPY pg_hba.conf /etc/postgresql/15/main/pg_hba.conf RUN pip install -e . USER postgres RUN /etc/init.d/postgresql start && psql < /app/db/create.sql -RUN /etc/init.d/postgresql start && vulndb nvd ingest +RUN /etc/init.d/postgresql start && vulndb nvd ingest --scope cve RUN /etc/init.d/postgresql start && vulndb osv ingest EXPOSE 5000 ENTRYPOINT [ "/app/vulndb.sh" ] diff --git a/pyproject.toml b/pyproject.toml index 837b1cdd..bf051346 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ datamodel-code-generator = ">=0.21.0" rich = "^13.4.2" univers = "^30.10.0" cpe = "^1.2.1" +requests = "^2.30.0" [build-system]