diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py b/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py
index 77364a385..2ea96343f 100644
--- a/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py
+++ b/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py
@@ -180,6 +180,23 @@ def get_introspector_report_url_debug_info(project_name, datestr):
datestr) + "all_debug_info.json"
+def get_introspector_report_url_fuzzer_log_file(project_name, datestr, fuzzer):
+ return get_introspector_report_url_base(
+ project_name, datestr) + f"fuzzerLogFile-{fuzzer}.data.yaml"
+
+
+def get_introspector_report_url_fuzzer_program_data(project_name, datestr,
+ program_data_filename):
+ return get_introspector_report_url_base(project_name,
+ datestr) + program_data_filename
+
+
+def get_introspector_report_url_fuzzer_coverage_urls(project_name, datestr,
+ coverage_files):
+ prefix = get_introspector_report_url_base(project_name, datestr)
+ return [prefix + ff for ff in coverage_files]
+
+
def extract_introspector_debug_info(project_name, date_str):
debug_data_url = get_introspector_report_url_debug_info(
project_name, date_str.replace("-", ""))
@@ -281,6 +298,59 @@ def get_fuzzer_code_coverage_summary(project_name, datestr, fuzzer):
return None
+MAGNITUDES = {
+ "k": 10**(3 * 1),
+ "M": 10**(3 * 2),
+ "G": 10**(3 * 3),
+ "T": 10**(3 * 4),
+ "P": 10**(3 * 5),
+ "E": 10**(3 * 6),
+ "Z": 10**(3 * 7),
+ "Y": 10**(3 * 8),
+}
+
+
+def get_fuzzer_corpus_size(project_name, datestr, fuzzer, introspector_report):
+ """Go through coverage reports to find the LLVMFuzzerTestOneInput function. The first hit count equals the number inputs found."""
+
+ if introspector_report["MergedProjectProfile"]["overview"][
+ "language"] != "c-cpp":
+ return None
+
+ metadata_files = introspector_report[fuzzer]["metadata-files"]
+
+ fuzzer_program_coverage_urls = get_introspector_report_url_fuzzer_coverage_urls(
+ project_name, datestr, metadata_files["coverage"])
+
+ for url in fuzzer_program_coverage_urls:
+ found = False
+ try:
+ cov_res = requests.get(url, timeout=20).text
+ for ll in cov_res.splitlines():
+ if found:
+ # Letters used is implemented here:
+ # https://github.com/llvm/llvm-project/blob/7569de527298a52618239ef68b9374a5c35c8b97/llvm/tools/llvm-cov/SourceCoverageView.cpp#L117
+ # Used from here:
+ # https://github.com/llvm/llvm-project/blob/35ed9a32d58bc8cbace31dc7c3bba79d0e3a9256/llvm/tools/llvm-cov/SourceCoverageView.h#L269
+ try:
+ count_str = ll.split("|")[1].strip()
+ magnitude_char = count_str[-1]
+ if magnitude_char.isalpha():
+ magnitude = MAGNITUDES[magnitude_char]
+ count = float(count_str[:-1])
+ else:
+ magnitude = 1
+ count = float(count_str)
+ return int(magnitude * count)
+ except:
+ # Something went wrong, maybe another file has correct data.
+ break
+ if ll == "LLVMFuzzerTestOneInput:":
+ found = True
+ except:
+ return None
+
+
def extract_new_introspector_functions(project_name, date_str):
introspector_functions_url = get_introspector_report_url_all_functions(
project_name, date_str.replace("-", ""))
@@ -372,7 +442,7 @@ def extract_introspector_report(project_name, date_str):
introspector_report_url = get_introspector_report_url_report(
project_name, date_str.replace("-", ""))
- # Read the introspector atifact
+ # Read the introspector artifact
try:
raw_introspector_json_request = requests.get(introspector_summary_url,
timeout=10)
diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py b/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py
index f9b9e1813..762cfd299 100644
--- a/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py
+++ b/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py
@@ -325,7 +325,7 @@ def extract_code_coverage_data(code_coverage_summary):
return line_total_summary
-def prepare_code_coverage_dict(
+def prepare_code_coverage_data(
code_coverage_summary, project_name: str, date_str: str,
project_language: str) -> Optional[Dict[str, Any]]:
"""Gets coverage URL and line coverage total of a project"""
@@ -472,7 +472,7 @@ def extract_local_project_data(project_name, oss_fuzz_path,
project_name
}
- code_coverage_data_dict = prepare_code_coverage_dict(
+ code_coverage_data_dict = prepare_code_coverage_data(
code_coverage_summary, project_name, '', project_language)
if cov_fuzz_stats is not None:
@@ -737,7 +737,7 @@ def extract_project_data(project_name, date_str, should_include_details,
'project_name': project_name
}
- code_coverage_data_dict = prepare_code_coverage_dict(
+ code_coverage_data_dict = prepare_code_coverage_data(
code_coverage_summary, project_name, date_str, project_language)
per_fuzzer_cov = {}
@@ -748,10 +748,19 @@ def extract_project_data(project_name, date_str, should_include_details,
amount_of_fuzzers = len(all_fuzzers)
for ff in all_fuzzers:
+ try:
+ fuzzer_corpus_size = oss_fuzz.get_fuzzer_corpus_size(
+ project_name, date_str.replace("-", ""), ff,
+ introspector_report)
+ except:
+ fuzzer_corpus_size = None
+
try:
fuzzer_cov = oss_fuzz.get_fuzzer_code_coverage_summary(
project_name, date_str.replace("-", ""), ff)
fuzzer_cov_data = extract_code_coverage_data(fuzzer_cov)
+ if fuzzer_cov_data is not None:
+ fuzzer_cov_data['corpus_size'] = fuzzer_corpus_size
per_fuzzer_cov[ff] = fuzzer_cov_data
except:
pass
@@ -919,8 +928,36 @@ def extend_db_timestamps(db_timestamp, output_directory):
json.dump(existing_timestamps, f)
+def per_fuzzer_coverage_has_degraded(fuzzer_data: List[Dict[str, Any]],
+ project_name: str,
+ ff: str) -> List[Dict[str, str]]:
+ """Go through the fuzzer data and find coverage drops."""
+
+ def get_url(date):
+ report_url = oss_fuzz.get_fuzzer_code_coverage_summary_url(
+ project_name, date.replace('-', ''), ff)
+ report_url = report_url[:-len('summary.json')] + 'index.html'
+ return report_url
+
+ res = []
+ for yesterday, today in zip(fuzzer_data[:-1], fuzzer_data[1:]):
+ if yesterday['percentage'] - today[
+ 'percentage'] > FUZZER_COVERAGE_IS_DEGRADED:
+ res.append({
+ 'before_date': yesterday['date'],
+ 'before_url': get_url(yesterday['date']),
+ 'before_perc': yesterday['percentage'],
+ 'current_date': today['date'],
+ 'current_url': get_url(today['date']),
+ 'current_perc': today['percentage'],
+ })
+
+ return res
+
+
def per_fuzzer_coverage_analysis(project_name: str,
- coverages: Dict[str, List[Tuple[int, str]]],
+ per_fuzzer_data: Dict[str, List[Dict[str,
+ Any]]],
lost_fuzzers):
"""Go through the recent coverage results and combine them into a short summary.
Including an assessment if the fuzzer got worse over time.
@@ -932,34 +969,47 @@ def per_fuzzer_coverage_analysis(project_name: str,
# at per fuzzer coverage, which is should already be normalized to what
# can be reached.
# TODO What would be a good percentage to mark as coverage degradation,
- # taking 5% for now but should be observed, maybe per it should be
+ # taking 5% for now but should be observed, maybe it should be
# configurable per project as well.
results = {}
- for ff, data in coverages.items():
+ for ff, data in per_fuzzer_data.items():
if len(data) > 0:
- values = [dd[0] for dd in data]
- dates = [dd[1] for dd in data]
- latest_date_with_value = next(dd[1] for dd in reversed(data)
- if dd[0] is not None)
+ percentages = [dd['percentage'] for dd in data]
+ dates = [dd['date'] for dd in data]
+ totals = [dd['total'] for dd in data]
+ covered = [dd['covered'] for dd in data]
+ corpus_size = [dd['corpus_size'] for dd in data]
+ latest_date_with_value = next(dd['date'] for dd in reversed(data)
+ if dd['percentage'] is not None)
if latest_date_with_value is not None:
report_url = oss_fuzz.get_fuzzer_code_coverage_summary_url(
project_name, latest_date_with_value.replace('-', ''), ff)
report_url = report_url[:-len('summary.json')] + 'index.html'
else:
report_url = None
- max_cov = max(values[:-1], default=0)
- avg_cov = round(statistics.fmean(values), 2)
- current = values[-1]
+ max_cov = max(percentages[:-1], default=0)
+ avg_cov = round(statistics.fmean(percentages), 2)
+ current = percentages[-1]
+ try:
+ days_degraded = per_fuzzer_coverage_has_degraded(
+ data, project_name, ff)
+ except:
+ days_degraded = []
results[ff] = {
'report_url': report_url,
'report_date': latest_date_with_value,
- 'coverages_values': values,
+ 'hashed_name': str(hash(ff)),
+ 'coverages_perc': percentages,
+ 'coverages_totals': totals,
+ 'coverages_covered': covered,
+ 'coverages_corpus': corpus_size,
'coverages_dates': dates,
'max': max_cov,
'avg': avg_cov,
'current': current,
- 'has_degraded':
+ 'max_has_degraded':
(max_cov - current) > FUZZER_COVERAGE_IS_DEGRADED,
+ 'days_degraded': days_degraded,
'got_lost': ff in lost_fuzzers,
}
return results
@@ -999,7 +1049,18 @@ def calculate_recent_results(projects_with_new_results, timestamps,
except:
perc = 0
- per_fuzzer_coverages[ff].append((perc, do))
+ per_fuzzer_coverages[ff].append({
+ 'corpus_size':
+ cov_data['corpus_size'],
+ 'covered':
+ cov_data['covered'],
+ 'total':
+ cov_data['count'],
+ 'percentage':
+ perc,
+ 'date':
+ do
+ })
except:
continue
@@ -1411,6 +1472,7 @@ def setup_webapp_cache() -> None:
os.mkdir("extracted-db-archive")
db_archive.extractall("extracted-db-archive")
+
logger.info("Extracted it all")
# Copy over the files
diff --git a/tools/web-fuzzing-introspection/app/webapp/models.py b/tools/web-fuzzing-introspection/app/webapp/models.py
index 67f3f04b8..8dc68ade0 100644
--- a/tools/web-fuzzing-introspection/app/webapp/models.py
+++ b/tools/web-fuzzing-introspection/app/webapp/models.py
@@ -46,7 +46,8 @@ def has_introspector(self) -> bool:
return self.introspector_data is not None
def has_recent_results(self) -> bool:
- return self.recent_results is not None
+ return self.recent_results is not None and sum(
+ len(ff) for ff in self.recent_results) > 0
class DBTimestamp:
diff --git a/tools/web-fuzzing-introspection/app/webapp/templates/project-profile.html b/tools/web-fuzzing-introspection/app/webapp/templates/project-profile.html
index 6874be5fc..4c8c3a654 100644
--- a/tools/web-fuzzing-introspection/app/webapp/templates/project-profile.html
+++ b/tools/web-fuzzing-introspection/app/webapp/templates/project-profile.html
@@ -1,433 +1,537 @@
-{% extends "base.html" %}
-{% block content %}
-
-
-
-
-
-
-
-
-
-
-
- Language |
- {{ project.language }}
- |
-
- {{page_main_name}} project |
- link |
-
- {% if project_repo %}
-
- Project repository |
- link |
-
- {% endif %}
-
- Build status: Fuzzers |
-
- {% if project_build_status.fuzz_build_status == false %}
- failing:
- {% else %}
- succeeding:
- {% endif %}
- Build log
- |
-
-
- Build status: Code coverage |
-
- {% if project_build_status.coverage_build_status == false %}
- failing:
- {% else %}
- succeeding:
- {% endif %}
- Build log
- |
-
-
- Build status: Fuzz Introspector |
-
- {% if project_build_status.introspector_build_status == false %}
- failing:
- {% else %}
- succeeding:
- {% endif %}
- Build log |
-
-
- Fuzzer count |
- {% if latest_statistics != None %}
- {{latest_statistics.fuzzer_count }} |
- {% else %}
- 0 |
- {% endif %}
-
-
- Lines of code |
-
- {% if project.coverage_data is not none %}
- {{ project.coverage_data.line_coverage.count }}
- {% elif latest_statistics != None and latest_statistics.coverage_data != None %}
- {{ latest_statistics.coverage_data.line_coverage.count }}
- {% else %}
- N/A
- {% endif %}
- |
-
-
- Lines covered |
-
- {% if project.coverage_data is not none %}
- {{ project.coverage_data.line_coverage.covered }}
- {% elif latest_statistics != None and latest_statistics.coverage_data != None %}
- {{ latest_statistics.coverage_data.line_coverage.covered }}
- {% else %}
- N/A
- {% endif %}
- |
-
-
- Code coverage |
-
- {% if has_project_details and project.coverage_data != None %}
- {{ '%0.2f' % project.coverage_data.line_coverage.percent |float}}%
- {% elif latest_statistics != None and latest_statistics.coverage_data != None %}
- {{ latest_statistics.coverage_data.line_coverage.percent }}%
- {% else %}
- N/A
- {% endif %}
- |
-
-
- Code coverage report |
- {% if has_project_details %}
- Report link |
- {% elif latest_coverage_report != None %}
- (Latest report from {{coverage_date}}) Report link |
- {% else %}
- N/A |
- {% endif %}
-
-
- Static reachability |
- {% if project.has_introspector() %}
- {{ '%0.2f' % project.introspector_data.static_reachability |float }}% |
- {% else %}
- N/A |
- {% endif %}
-
-
- Fuzz Introspector report |
- {% if (latest_fuzz_introspector_report and latest_fuzz_introspector_report != "N/A") %}
- Report link (from {{ latest_introspector_datestr }}) |
- {% else %}
- N/A |
- {% endif %}
-
-
-
-
- {% if project.has_recent_results() %}
-
-
-
- Fuzzer |
- Code coverage (lines) |
- Latest Report |
- Comments |
-
-
- {% for fuzzer, fuzzer_data in project.recent_results.items()
- |sort(attribute='0')
- |sort(reverse=true, attribute='1.got_lost,1.has_degraded')
- %}
-
-
- {{ fuzzer }}
- |
-
-
- {{ fuzzer_data['current'] }}% (avg: {{ fuzzer_data['avg'] }}%, max: {{ fuzzer_data['max'] }}%)
-
- |
-
- {{ fuzzer_data['report_date'] }}
- |
-
- {% if fuzzer_data['got_lost'] %}
- Fuzzer no longer available!
- {% endif %}
- {% if fuzzer_data['has_degraded'] %}
- Coverage has degraded!
- {% endif %}
- |
-
- {%endfor%}
-
-
-
- {% endif %}
-
-
-
-
-
-
Historical Progression
-
-
-
-
-
-
- {% if project.has_introspector() %}
-
-
-
-
- {% else %}
-
-
-
Missing Introspector data
- Some graphs are not shown as there is no Fuzz Introspector data available. Please see indexing page for the relevant build logs. Once the build is working additionals graphs will be displayed.
-
-
- {% endif %}
-
-
-
- {% if project.has_recent_results() %}
-
-
Per Fuzzer Progression
-
-
-
- {% endif %}
-
-
-
- {% if project.has_introspector() %}
-
-
-
- Functions of interest to fuzz
-
-
- This section outlines functions that may be of interest to fuzz.
- They are based on ranking functions that have a lot of complexity but currently
- exhibit low code coverage. The complexity is calculated based on the function itself
- as well as the functions called by the given function, i.e. the tree of code
- that the function triggers.
-
-
-
- This is only a minor amount of introspection information available for this project. Please
- consult the Fuzz Introspector report
- for more information, e.g. the introspection table of all functions in the target
- project available here.
-
-
-
-
-
-
-
-
-
- Function name |
- Function source file |
- Accumulated cyclomatic complexity |
- Code coverage |
-
-
-
- {% for func_of_interest in functions_of_interest %}
-
- {{func_of_interest.function_name}} |
- {{func_of_interest.source_file}} |
- {{func_of_interest.complexity}} |
- {{func_of_interest.code_coverage}}% |
-
- {% endfor %}
-
-
-
-
-
-
- {% endif %}
-
-
-
-
-
-
-
-
-
- {% if project.has_introspector() %}
-
- {% endif %}
-
-
-
-
-
-
-
-{% endblock %}
+{% extends "base.html" %}
+{% block content %}
+
+
+
+
+
+
+
+
+
+
+
+ Language |
+ {{ project.language }}
+ |
+
+ {{page_main_name}} project |
+ link |
+
+ {% if project_repo %}
+
+ Project repository |
+ link |
+
+ {% endif %}
+
+ Build status: Fuzzers |
+
+ {% if project_build_status.fuzz_build_status == false %}
+ failing:
+ {% else %}
+ succeeding:
+ {% endif %}
+ Build log
+ |
+
+
+ Build status: Code coverage |
+
+ {% if project_build_status.coverage_build_status == false %}
+ failing:
+ {% else %}
+ succeeding:
+ {% endif %}
+ Build log
+ |
+
+
+ Build status: Fuzz Introspector |
+
+ {% if project_build_status.introspector_build_status == false %}
+ failing:
+ {% else %}
+ succeeding:
+ {% endif %}
+ Build log |
+
+
+ Fuzzer count |
+ {% if latest_statistics != None %}
+ {{latest_statistics.fuzzer_count }} |
+ {% else %}
+ 0 |
+ {% endif %}
+
+
+ Lines of code |
+
+ {% if project.coverage_data is not none %}
+ {{ project.coverage_data.line_coverage.count }}
+ {% elif latest_statistics != None and latest_statistics.coverage_data != None %}
+ {{ latest_statistics.coverage_data.line_coverage.count }}
+ {% else %}
+ N/A
+ {% endif %}
+ |
+
+
+ Lines covered |
+
+ {% if project.coverage_data is not none %}
+ {{ project.coverage_data.line_coverage.covered }}
+ {% elif latest_statistics != None and latest_statistics.coverage_data != None %}
+ {{ latest_statistics.coverage_data.line_coverage.covered }}
+ {% else %}
+ N/A
+ {% endif %}
+ |
+
+
+ Code coverage |
+
+ {% if has_project_details and project.coverage_data != None %}
+ {{ '%0.2f' % project.coverage_data.line_coverage.percent |float}}%
+ {% elif latest_statistics != None and latest_statistics.coverage_data != None %}
+ {{ latest_statistics.coverage_data.line_coverage.percent }}%
+ {% else %}
+ N/A
+ {% endif %}
+ |
+
+
+ Code coverage report |
+ {% if has_project_details %}
+ Report link |
+ {% elif latest_coverage_report != None %}
+ (Latest report from {{coverage_date}}) Report link |
+ {% else %}
+ N/A |
+ {% endif %}
+
+
+ Static reachability |
+ {% if project.has_introspector() %}
+ {{ '%0.2f' % project.introspector_data.static_reachability |float }}% |
+ {% else %}
+ N/A |
+ {% endif %}
+
+
+ Fuzz Introspector report |
+ {% if (latest_fuzz_introspector_report and latest_fuzz_introspector_report != "N/A") %}
+ Report link (from {{ latest_introspector_datestr }}) |
+ {% else %}
+ N/A |
+ {% endif %}
+
+
+
+
+ {% if project.has_recent_results() %}
+
+ {% endif %}
+
+
+
+
+
+
Historical Progression
+
+
+
+
+
+
+
+ {% if project.has_introspector() %}
+
+
+
+
+
+ {% else %}
+
+
+
Missing Introspector data
+ Some graphs are not shown as there is no Fuzz Introspector data available. Please see indexing page for the relevant build logs. Once the build is working additionals graphs will be displayed.
+
+
+ {% endif %}
+
+
+
+ {% if project.has_recent_results() %}
+
+
+
Per Fuzzer Progression
+
+ This section shows graphs for the coverage results per fuzz target over the past 30 days. Included are the
+ coverage percentage,
+ total number of lines,
+ covered number of lines, and the number of
+ coverage inputs.
+
+
+
+
+
+ {% endif %}
+
+
+
+ {% if project.has_introspector() %}
+
+
+
+ Functions of interest to fuzz
+
+
+ This section outlines functions that may be of interest to fuzz.
+ They are based on ranking functions that have a lot of complexity but currently
+ exhibit low code coverage. The complexity is calculated based on the function itself
+ as well as the functions called by the given function, i.e. the tree of code
+ that the function triggers.
+
+
+
+ This is only a minor amount of introspection information available for this project. Please
+ consult the Fuzz Introspector report
+ for more information, e.g. the introspection table of all functions in the target
+ project available here.
+
+
+
+
+
+
+
+
+
+ Function name |
+ Function source file |
+ Accumulated cyclomatic complexity |
+ Code coverage |
+
+
+
+ {% for func_of_interest in functions_of_interest %}
+
+ {{func_of_interest.function_name}} |
+ {{func_of_interest.source_file}} |
+ {{func_of_interest.complexity}} |
+ {{func_of_interest.code_coverage}}% |
+
+ {% endfor %}
+
+
+
+
+
+
+ {% endif %}
+
+
+
+
+
+
+
+
+
+ {% if project.has_introspector() %}
+
+ {% endif %}
+
+
+
+
+
+
+
+{% endblock %}
+