diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py b/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py index e7ddf8b52..b53ebe080 100644 --- a/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py +++ b/tools/web-fuzzing-introspection/app/static/assets/db/oss_fuzz.py @@ -32,6 +32,11 @@ def get_introspector_report_url_summary(project_name, datestr): datestr) + "summary.json" +def get_introspector_report_url_branch_blockers(project_name, datestr): + return get_introspector_report_url_base(project_name, + datestr) + "branch-blockers.json" + + def get_introspector_report_url_all_functions(project_name, datestr): return get_introspector_report_url_base( project_name, datestr) + "all-fuzz-introspector-functions.json" @@ -152,6 +157,24 @@ def extract_introspector_light_all_files(project_name, date_str): return all_urls +def extract_introspector_branch_blockers(project_name, date_str): + introspector_branch_blockers_url = get_introspector_report_url_branch_blockers( + project_name, date_str.replace("-", "")) + + # Read the introspector atifact + try: + raw_introspector_json_request = requests.get( + introspector_branch_blockers_url, timeout=10) + except: + return None + try: + branch_blockers = json.loads(raw_introspector_json_request.text) + except: + return None + + return branch_blockers + + def get_introspector_type_map_url_summary(project_name, datestr): return get_introspector_report_url_base( project_name, datestr) + "all-friendly-debug-types.json" @@ -228,6 +251,16 @@ def extract_local_introspector_function_list(project_name, oss_fuzz_folder): return function_list +def extract_local_introspector_branch_blockers(project_name, oss_fuzz_folder): + summary_json = os.path.join(oss_fuzz_folder, 'build', 'out', project_name, + 'inspector', 'branch-blockers.json') + if not os.path.isfile(summary_json): + return {} + with open(summary_json, 'r') as f: + json_dict = json.load(f) + return json_dict + + def extract_local_introspector_constructor_list(project_name, oss_fuzz_folder): summary_json = os.path.join(oss_fuzz_folder, 'build', 'out', project_name, 'inspector', diff --git a/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py b/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py index 6333ed592..49dc63478 100644 --- a/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py +++ b/tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py @@ -171,24 +171,16 @@ def save_type_map(debug_report, project_name): json.dump(debug_report, report_fd) -def extract_and_refine_branch_blockers(introspector_report, project_name): +def extract_and_refine_branch_blockers(branch_blockers, project_name): branch_pairs = list() - for key in introspector_report: - if key == "MergedProjectProfile" or key == 'analyses': - continue - # Fuzzer-specific dictionary, get the contents of it. - val = introspector_report[key] - if not isinstance(val, dict): - continue - - branch_blockers = val.get('branch_blockers', None) - if branch_blockers is None or not isinstance(branch_blockers, list): - continue + if not isinstance(branch_blockers, dict): + return branch_pairs - for branch_blocker in branch_blockers: - function_blocked = branch_blocker.get('function_name', None) - blocked_unique_not_covered_complexity = branch_blocker.get( + for _, blockers in branch_blockers.items(): + for blocker in blockers: + function_blocked = blocker.get('function_name', None) + blocked_unique_not_covered_complexity = blocker.get( 'blocked_unique_not_covered_complexity', None) if blocked_unique_not_covered_complexity < 5: continue @@ -205,11 +197,11 @@ def extract_and_refine_branch_blockers(introspector_report, project_name): 'blocked_runtime_coverage': blocked_unique_not_covered_complexity, 'source_file': - branch_blocker.get('source_file', "N/A"), + blocker.get('source_file', "N/A"), 'linenumber': - branch_blocker.get('branch_line_number', -1), + blocker.get('branch_line_number', -1), 'blocked_unique_functions': - branch_blocker.get('blocked_unique_functions', []) + blocker.get('blocked_unique_functions', []) }) return branch_pairs @@ -357,6 +349,8 @@ def extract_local_project_data(project_name, oss_fuzz_path, # Default set to c++ as this is OSS-Fuzz's default. project_language = 'c++' + branch_blockers = oss_fuzz.extract_local_introspector_branch_blockers( + project_name, oss_fuzz_path) code_coverage_summary = oss_fuzz.get_local_code_coverage_summary( project_name, oss_fuzz_path) cov_fuzz_stats = oss_fuzz.get_local_code_coverage_stats( @@ -443,8 +437,11 @@ def extract_local_project_data(project_name, oss_fuzz_path, all_constructor_list, '') annotated_cfg = extract_and_refine_annotated_cfg(introspector_report) + branch_pairs = extract_and_refine_branch_blockers(branch_blockers, + project_name) # Dump things we dont want to accummulate. - #save_branch_blockers(branch_pairs, project_name) + # save_branch_blockers(branch_pairs, project_name) + try: project_repository = oss_fuzz.try_to_get_project_repository( project_name) @@ -570,6 +567,9 @@ def extract_project_data(project_name, date_str, should_include_details, introspector_report_url = oss_fuzz.get_introspector_report_url_report( project_name, date_str.replace("-", "")) + branch_blockers = oss_fuzz.extract_introspector_branch_blockers( + project_name, date_str.replace("-", "")) + test_files = oss_fuzz.extract_introspector_test_files( project_name, date_str.replace("-", "")) if test_files: @@ -730,10 +730,10 @@ def extract_project_data(project_name, date_str, should_include_details, annotated_cfg = extract_and_refine_annotated_cfg( introspector_report) branch_pairs = extract_and_refine_branch_blockers( - introspector_report, project_name) + branch_blockers, project_name) # Dump things we dont want to accummulate. - save_branch_blockers(branch_pairs, project_name) + # save_branch_blockers(branch_pairs, project_name) # Extract type definition typedef_list = oss_fuzz.extract_introspector_typedef( diff --git a/tools/web-fuzzing-introspection/app/webapp/routes.py b/tools/web-fuzzing-introspection/app/webapp/routes.py index 1b30b0a9d..42f54aafc 100644 --- a/tools/web-fuzzing-introspection/app/webapp/routes.py +++ b/tools/web-fuzzing-introspection/app/webapp/routes.py @@ -1485,6 +1485,9 @@ def branch_blockers(args): ## Example 1: - `project`: `tinyxml2` + + **Note:** + This API endpoint currently does not return branch blocker data due to ongoing considerations regarding data storage. Please refer to [#2185](https://github.com/ossf/fuzz-introspector/issues/2185) [#2183](https://github.com/ossf/fuzz-introspector/pull/2183) for details. """ project_name = args.get('project', '') if not project_name: