From d262f7a4c932b0e823b3b9d979aa1122c4590078 Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 23 Nov 2021 22:00:41 -0700 Subject: [PATCH 1/4] first shot at adding allow list Signed-off-by: vsoch --- Dockerfile | 2 +- README.md | 13 +++++++++-- allowlist.yaml | 10 +++++++++ requirements.txt | 1 + stools/clair/__init__.py | 12 +++++++++- stools/clair/api.py | 47 +++++++++++++++++++++++++++++++++++----- 6 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 allowlist.yaml diff --git a/Dockerfile b/Dockerfile index d35020c..46dad71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM singularityware/singularity:v3.2.1-slim as base ################################################################################ # -# Copyright (C) 2019 Vanessa Sochat. +# Copyright (C) 2019-2021 Vanessa Sochat. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License as published by diff --git a/README.md b/README.md index 6b9c9ca..69d02fd 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ This experiment is based on early discussion in [this thread](https://github.com You'll need to first clone the repository: ```bash -git clone https://github.com/singularityhub/stools -cd stools +$ git clone https://github.com/singularityhub/stools +$ cd stools ``` ### Build Containers @@ -100,6 +100,15 @@ http://people.ubuntu.com/~ubuntu-security/cve/CVE-2016-9843 The crc32_big function in crc32.c in zlib 1.2.8 might allow context-dependent attackers to have unspecified impact via vectors involving big-endian CRC calculation. ``` +To include an allowlist, e.g., [allowlist.yaml](allowlist.yaml) you can do: + +```bash +$ docker exec -it clair-scanner sclair --allowlist allowlist.yaml singularity-images_latest.sif +``` + +You'll notice the previous last entry is different, because it was removed. Currently, we just match CVE names (and don't do +further parsing) but this can be tweaked if desired. + ### Save a Report However, if you want to save a report to file (json), you can add the `--report` argument diff --git a/allowlist.yaml b/allowlist.yaml new file mode 100644 index 0000000..637b5b5 --- /dev/null +++ b/allowlist.yaml @@ -0,0 +1,10 @@ +generalallowlist: # Approve CVE for any image + CVE-2017-6055: XML + CVE-2017-5586: OpenText + CVE-2019-13627: "" +images: + ubuntu: # Approve CVE only for ubuntu image, regardles of the version. If it is a private registry with a custom port registry:777/ubuntu:tag this won't work due to a bug. + CVE-2017-5230: Java + CVE-2017-5230: XSX + alpine: + CVE-2017-3261: SE diff --git a/requirements.txt b/requirements.txt index f20313b..7bdf7cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ aiohttp==3.7.4 requests>=2.20.0 +pyaml diff --git a/stools/clair/__init__.py b/stools/clair/__init__.py index dc4564e..c598bb2 100644 --- a/stools/clair/__init__.py +++ b/stools/clair/__init__.py @@ -64,6 +64,12 @@ def get_parser(): help="save Clair reports to chosen directory", ) + parser.add_argument( + "--allowlist", + default=None, + help="include a yaml allow list (example in stools repository)", + ) + parser.add_argument( "--no-print", dest="no_print", @@ -147,6 +153,10 @@ def help(retval=0): # Local Server webroot = "/var/www/images" + # If we have an allowlist, make sure it exists + if args.allowlist and not os.path.exists(args.allowlist): + sys.exit("%s does not exist." % args.allowlist) + # Start the server and serve static files from root if args.server: @@ -183,7 +193,7 @@ def help(retval=0): # 4. Generate report print("3. Generating report!") - report = clair.report(os.path.basename(image)) + report = clair.report(os.path.basename(image), args.allowlist) if args.report_location: fpath = os.path.join( args.report_location, diff --git a/stools/clair/api.py b/stools/clair/api.py index 733b305..9ba8be0 100644 --- a/stools/clair/api.py +++ b/stools/clair/api.py @@ -19,11 +19,12 @@ import requests +import yaml import os import sys -class Clair(object): +class Clair: """the ClairOS security scanner to scan Docker layers""" def __init__(self, host, port, api_version="v1"): @@ -50,22 +51,56 @@ def scan(self, targz_url, name): print("Error creating %s at %s" % (data["Path"], url)) sys.exit(1) - def report(self, name): + def report(self, name, allowlist=None): """generate a report for an image of interest. The name should correspond to the same name used when adding the layer... - - Parameters - ========== """ url = os.path.join(self.url, "layers", name) response = requests.get(url, params={"features": True, "vulnerabilities": True}) if response.status_code == 200: - return response.json() + hits = response.json() + if allowlist: + hits = self.apply_allowlist(allowlist, hits) + return hits else: print("Error with %s" % url) sys.exit(1) + def apply_allowlist(self, filename, hits): + """ + Apply an allowlist, meaning a yaml of vulnerabilities to ignore / remove. + """ + with open(filename, "r") as fd: + allow = yaml.load(fd.read(), Loader=yaml.SafeLoader) + + # No results? + if "Layer" not in hits: + return hits + + # General allowlist + general = set(allow.get("generalallowlist", {})) + + for image, cves in allow["images"].items(): + + # Just match based on list of names (we might want to extend this) + cves = set(cves) + if not hits["Layer"]["NamespaceName"].startswith(image): + continue + for feature in hits["Layer"].get("Features", []): + if "Vulnerabilities" not in feature: + continue + vulns = [] + + # For a vulnerability, if it's not in allow list, add + for vuln in feature["Vulnerabilities"]: + if vuln["Name"] in cves or vuln["Name"] in general: + print("Allowlist: skipping %s" % vuln["Name"]) + continue + vulns.append(vuln) + feature["Vulnerabilities"] = vulns + return hits + def ping(self): """ping serves as a health check. If healthy, will return True. We do this because the user is starting Clair as From b50ade5066a813bb41749351437f65649f078a07 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sat, 1 Jan 2022 20:33:08 -0700 Subject: [PATCH 2/4] update license headers Signed-off-by: vsoch --- Dockerfile | 2 +- requirements.txt | 1 + stools/clair/__init__.py | 2 +- stools/clair/api.py | 2 +- stools/clair/image.py | 2 +- stools/clair/server/main.py | 2 +- stools/clair/server/routes.py | 2 +- stools/clair/server/views.py | 2 +- stools/utils.py | 2 +- stools/version.py | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 46dad71..ba4d357 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM singularityware/singularity:v3.2.1-slim as base ################################################################################ # -# Copyright (C) 2019-2021 Vanessa Sochat. +# Copyright (C) 2019-2022 Vanessa Sochat. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License as published by diff --git a/requirements.txt b/requirements.txt index 7bdf7cd..9101cb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ aiohttp==3.7.4 requests>=2.20.0 pyaml +IPython diff --git a/stools/clair/__init__.py b/stools/clair/__init__.py index c598bb2..54516ff 100644 --- a/stools/clair/__init__.py +++ b/stools/clair/__init__.py @@ -2,7 +2,7 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/stools/clair/api.py b/stools/clair/api.py index 9ba8be0..ce4f647 100644 --- a/stools/clair/api.py +++ b/stools/clair/api.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/stools/clair/image.py b/stools/clair/image.py index ca38636..d55990f 100644 --- a/stools/clair/image.py +++ b/stools/clair/image.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/stools/clair/server/main.py b/stools/clair/server/main.py index 43cf3ea..0cf2cc7 100644 --- a/stools/clair/server/main.py +++ b/stools/clair/server/main.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/stools/clair/server/routes.py b/stools/clair/server/routes.py index 2e84bea..1dd98e8 100644 --- a/stools/clair/server/routes.py +++ b/stools/clair/server/routes.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/stools/clair/server/views.py b/stools/clair/server/views.py index 62ecbaa..7aacf05 100644 --- a/stools/clair/server/views.py +++ b/stools/clair/server/views.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/stools/utils.py b/stools/utils.py index e83dee5..a3a01f9 100644 --- a/stools/utils.py +++ b/stools/utils.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/stools/version.py b/stools/version.py index a659ad0..4a8d0e8 100644 --- a/stools/version.py +++ b/stools/version.py @@ -1,6 +1,6 @@ """ -Copyright (C) 2018-2021 Vanessa Sochat. +Copyright (C) 2018-2022 Vanessa Sochat. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by From 7e63aa8899c36fbabcdafcecf8baff13d9398818 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sun, 2 Jan 2022 13:24:33 -0700 Subject: [PATCH 3/4] add printing of allowed and unallowed Signed-off-by: vsoch --- stools/clair/api.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/stools/clair/api.py b/stools/clair/api.py index ce4f647..0f527cc 100644 --- a/stools/clair/api.py +++ b/stools/clair/api.py @@ -90,15 +90,21 @@ def apply_allowlist(self, filename, hits): for feature in hits["Layer"].get("Features", []): if "Vulnerabilities" not in feature: continue + + # Keep record of vulns and allowed vulns = [] + allowed = [] # For a vulnerability, if it's not in allow list, add for vuln in feature["Vulnerabilities"]: if vuln["Name"] in cves or vuln["Name"] in general: print("Allowlist: skipping %s" % vuln["Name"]) + allowed.append(vuln) continue vulns.append(vuln) + feature["Vulnerabilities"] = vulns + feature["Allowed"] = allowed return hits def ping(self): @@ -132,13 +138,23 @@ def print(self, report): items = report["Layer"]["Features"] for item in items: - if "Vulnerabilities" in item: + + # Print a header given any items + if "Approved" in item or "Vulnerabilities" in item: print("%s - %s" % (item["Name"], item["Version"])) print("-" * len(item["Name"] + " - " + item["Version"])) - for v in item["Vulnerabilities"]: + + if "Approved" in item: + for v in item["Approved"]: print(v["Name"] + " (" + v["Severity"] + ")") print(v["Link"]) print(v["Description"]) print("\n") + if "Vulnerabilities" in item: + for v in item["Vulnerabilities"]: + print(v["Name"] + " (" + v["Severity"] + ") unapproved ") + print(v["Link"]) + print(v["Description"]) + print("\n") else: print("%s does not have any vulnerabilities!" % report["Layer"]["Name"]) From ed170ce69fd63e7cd7a1f435686b9c577fb689a4 Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 3 Jan 2022 12:52:56 -0700 Subject: [PATCH 4/4] fixing printing of allowed Signed-off-by: vsoch --- stools/clair/api.py | 60 ++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/stools/clair/api.py b/stools/clair/api.py index 0f527cc..bd40703 100644 --- a/stools/clair/api.py +++ b/stools/clair/api.py @@ -55,7 +55,6 @@ def report(self, name, allowlist=None): """generate a report for an image of interest. The name should correspond to the same name used when adding the layer... """ - url = os.path.join(self.url, "layers", name) response = requests.get(url, params={"features": True, "vulnerabilities": True}) if response.status_code == 200: @@ -87,13 +86,21 @@ def apply_allowlist(self, filename, hits): cves = set(cves) if not hits["Layer"]["NamespaceName"].startswith(image): continue + + # Don't continue if no features + if not hits["Layer"].get("Features", []): + continue + + # Keep list of updated features + updated = [] for feature in hits["Layer"].get("Features", []): if "Vulnerabilities" not in feature: + updated.append(feature) continue # Keep record of vulns and allowed vulns = [] - allowed = [] + allowed = feature.get("Allowed", []) # For a vulnerability, if it's not in allow list, add for vuln in feature["Vulnerabilities"]: @@ -105,6 +112,9 @@ def apply_allowlist(self, filename, hits): feature["Vulnerabilities"] = vulns feature["Allowed"] = allowed + updated.append(feature) + + hits["Layer"]["Features"] = updated return hits def ping(self): @@ -134,27 +144,27 @@ def ping(self): def print(self, report): """print the report items""" - if "Features" in report["Layer"]: - items = report["Layer"]["Features"] - - for item in items: - - # Print a header given any items - if "Approved" in item or "Vulnerabilities" in item: - print("%s - %s" % (item["Name"], item["Version"])) - print("-" * len(item["Name"] + " - " + item["Version"])) - - if "Approved" in item: - for v in item["Approved"]: - print(v["Name"] + " (" + v["Severity"] + ")") - print(v["Link"]) - print(v["Description"]) - print("\n") - if "Vulnerabilities" in item: - for v in item["Vulnerabilities"]: - print(v["Name"] + " (" + v["Severity"] + ") unapproved ") - print(v["Link"]) - print(v["Description"]) - print("\n") - else: + features = report["Layer"].get("Features", []) + if not features: print("%s does not have any vulnerabilities!" % report["Layer"]["Name"]) + return + + for item in features: + + # Print a header given any items + if "Allowed" in item or "Vulnerabilities" in item: + print("%s - %s" % (item["Name"], item["Version"])) + print("-" * len(item["Name"] + " - " + item["Version"])) + + if "Allowed" in item: + for v in item["Allowed"]: + print(v["Name"] + " (" + v["Severity"] + ")") + print(v["Link"]) + print(v["Description"]) + print("\n") + if "Vulnerabilities" in item: + for v in item["Vulnerabilities"]: + print(v["Name"] + " (" + v["Severity"] + ") unapproved ") + print(v["Link"]) + print(v["Description"]) + print("\n")