From db9d195f89303a5eef1d637a298a87cea760eea0 Mon Sep 17 00:00:00 2001 From: Ezequiel Fuentes Martin Date: Thu, 28 May 2026 18:24:23 -0400 Subject: [PATCH] Moved main logic to orchestrator. Exposing the cli capabilities to the API --- src/cliparser.rs | 81 ------- src/lib.rs | 5 + src/main.rs | 386 +-------------------------------- src/orchestrator.rs | 504 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 512 insertions(+), 464 deletions(-) delete mode 100644 src/cliparser.rs create mode 100644 src/orchestrator.rs diff --git a/src/cliparser.rs b/src/cliparser.rs deleted file mode 100644 index 523ce30c4..000000000 --- a/src/cliparser.rs +++ /dev/null @@ -1,81 +0,0 @@ -use clap::{CommandFactory, Parser}; - -#[derive(Debug, Parser)] -#[command(author, version, about, long_about = None)] -pub struct CliArgs { - /// List supported signatures and extractors - #[arg(short = 'L', long)] - pub list: bool, - - /// Read data from standard input - #[arg(short, long)] - pub stdin: bool, - - /// Supress normal stdout output - #[arg(short, long)] - pub quiet: bool, - - /// During recursive extraction display *all* results - #[arg(short, long)] - pub verbose: bool, - - /// Automatically extract known file types - #[arg(short, long)] - pub extract: bool, - - /// Carve both known and unknown file contents to disk - #[arg(short, long)] - pub carve: bool, - - /// Recursively scan extracted files - #[arg(short = 'M', long)] - pub matryoshka: bool, - - /// Search for all signatures at all offsets - #[arg(short = 'a', long)] - pub search_all: bool, - - /// Generate an entropy graph with Plotly - #[arg(short = 'E', long, conflicts_with = "extract")] - pub entropy: bool, - - /// Save entropy graph as a PNG file - #[arg(short, long)] - pub png: Option, - - /// Log JSON results to a file ('-' for stdout) - #[arg(short, long)] - pub log: Option, - - /// Manually specify the number of threads to use - #[arg(short, long)] - pub threads: Option, - - /// Do no scan for these signatures - #[arg(short = 'x', long, value_delimiter = ',', num_args = 1..)] - pub exclude: Option>, - - /// Only scan for these signatures - #[arg(short = 'y', long, value_delimiter = ',', num_args = 1.., conflicts_with = "exclude")] - pub include: Option>, - - /// Extract files/folders to a custom directory - #[arg(short, long, default_value = "extractions")] - pub directory: String, - - /// Path to the file to analyze - pub file_name: Option, -} - -pub fn parse() -> CliArgs { - let args = CliArgs::parse(); - - if std::env::args().len() == 1 { - CliArgs::command() - .print_help() - .expect("Failed to print help output"); - std::process::exit(0); - } - - args -} diff --git a/src/lib.rs b/src/lib.rs index c5151dd78..a52ae9dd3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,8 +18,13 @@ //! ``` mod binwalk; pub mod common; +mod display; +mod entropy; pub mod extractors; +mod json; mod magic; +pub mod orchestrator; pub mod signatures; pub mod structures; pub use binwalk::{AnalysisResults, Binwalk, BinwalkError}; +pub use orchestrator::{Orchestrator, OrchestratorArgs}; diff --git a/src/main.rs b/src/main.rs index 6a885e180..fc75decf2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,397 +1,17 @@ -use binwalk::AnalysisResults; -use log::{debug, error, info}; -use std::collections::VecDeque; -use std::panic; -use std::process; use std::process::ExitCode; -use std::sync::mpsc; -use std::thread; -use std::time; -use threadpool::ThreadPool; mod binwalk; -mod cliparser; mod common; mod display; mod entropy; mod extractors; mod json; mod magic; +mod orchestrator; mod signatures; mod structures; fn main() -> ExitCode { - // File name used when reading from stdin - const STDIN: &str = "stdin"; - - // Only use one thread if unable to auto-detect available core info - const DEFAULT_WORKER_COUNT: usize = 1; - - // Number of seconds to wait before printing debug progress info - const PROGRESS_INTERVAL: u64 = 30; - - // If this env var is set during extraction, the Binwalk.base_target_file symlink will - // be deleted at the end of extraction. - const BINWALK_RM_SYMLINK: &str = "BINWALK_RM_EXTRACTION_SYMLINK"; - - // Output directory for extracted files - let mut output_directory: Option = None; - - /* - * Maintain a queue of files waiting to be analyzed. - * Note that ThreadPool has its own internal queue so this may seem redundant, however, - * queuing a large number of jobs via the ThreadPool queue results in *massive* amounts - * of unecessary memory consumption, especially when recursively analyzing many files. - */ - let mut target_files = VecDeque::new(); - - // Statistics variables; keeps track of analyzed file count and total analysis run time - let mut file_count: usize = 0; - let run_time = time::Instant::now(); - let mut last_progress_interval = time::Instant::now(); - - // Initialize logging - env_logger::init(); - - // Process command line arguments - let mut cliargs = cliparser::parse(); - - // If --list was specified, just display a list of signatures and return - if cliargs.list { - display::print_signature_list(cliargs.quiet, &magic::patterns()); - return ExitCode::SUCCESS; - } - - // Set a dummy file name when reading from stdin - if cliargs.stdin { - cliargs.file_name = Some(STDIN.to_string()); - } - - let mut json_logger = json::JsonLogger::new(cliargs.log); - - // If entropy analysis was requested, generate the entropy graph and return - if cliargs.entropy { - display::print_plain(cliargs.quiet, "Calculating file entropy..."); - - if let Ok(entropy_results) = - entropy::plot(cliargs.file_name.unwrap(), cliargs.stdin, cliargs.png) - { - // Log entropy results to JSON file, if requested - json_logger.log(json::JSONType::Entropy(entropy_results.clone())); - json_logger.close(); - - display::println_plain(cliargs.quiet, "done."); - } else { - panic!("Entropy analysis failed!"); - } - - return ExitCode::SUCCESS; - } - - // If extraction or data carving was requested, we need to initialize the output directory - if cliargs.extract || cliargs.carve { - output_directory = Some(cliargs.directory); - } - - // Initialize binwalk - let binwalker = match binwalk::Binwalk::configure( - cliargs.file_name, - output_directory, - cliargs.include, - cliargs.exclude, - None, - cliargs.search_all, - ) { - Err(e) => { - error!("Binwalk initialization failed: {}", e.message); - return ExitCode::FAILURE; - } - Ok(bw) => bw, - }; - - // If the user specified --threads, honor that request; else, auto-detect available parallelism - let available_workers = cliargs.threads.unwrap_or_else(|| { - // Get CPU core info - match thread::available_parallelism() { - // In case of error use the default - Err(e) => { - error!("Failed to retrieve CPU core info: {e}"); - DEFAULT_WORKER_COUNT - } - Ok(coreinfo) => coreinfo.get(), - } - }); - - // Sanity check the number of available worker threads - if available_workers < 1 { - panic!("No available worker threads!"); - } - - // Initialize thread pool - debug!("Initializing thread pool with {available_workers} workers"); - let workers = ThreadPool::new(available_workers); - let (worker_tx, worker_rx) = mpsc::channel(); - - /* - * Set a custom panic handler. - * This ensures that when any thread panics, the default panic handler will be invoked - * _and_ the entire process will exit with an error code. - */ - let default_panic_handler = panic::take_hook(); - panic::set_hook(Box::new(move |panic_info| { - default_panic_handler(panic_info); - process::exit(-1); - })); - - debug!( - "Queuing initial target file: {}", - binwalker.base_target_file - ); - - // Queue the initial file path - target_files.insert(target_files.len(), binwalker.base_target_file.clone()); - - /* - * Main loop. - * Loop until all pending thread jobs are complete and there are no more files in the queue. - */ - while !target_files.is_empty() || workers.active_count() > 0 { - // If there are files waiting to be analyzed and there is at least one free thread in the pool - if !target_files.is_empty() && workers.active_count() < workers.max_count() { - // Get the next file path from the target_files queue - let target_file = target_files - .pop_front() - .expect("Failed to retrieve next file from the queue"); - - // Spawn a new worker for the new file - spawn_worker( - &workers, - binwalker.clone(), - target_file, - cliargs.stdin && file_count == 0, - cliargs.extract, - cliargs.carve, - worker_tx.clone(), - ); - } - - // Don't spin CPU cycles if there is no backlog of files to analyze - if target_files.is_empty() { - let sleep_time = time::Duration::from_millis(1); - thread::sleep(sleep_time); - } - - // Some debug info on analysis progress - if last_progress_interval.elapsed().as_secs() >= PROGRESS_INTERVAL { - info!( - "Status: active worker threads: {}/{}, files waiting in queue: {}", - workers.active_count(), - workers.max_count(), - target_files.len() - ); - last_progress_interval = time::Instant::now(); - } - - // Get response from a worker thread, if any - if let Ok(results) = worker_rx.try_recv() { - // Keep a tally of how many files have been analyzed - file_count += 1; - - // Log analysis results to JSON file - json_logger.log(json::JSONType::Analysis(results.clone())); - - // Nothing found? Nothing else to do for this file. - if results.file_map.is_empty() { - debug!("Found no results for file {}", results.file_path); - continue; - } - - // Print analysis results to screen - if should_display(&results, file_count, cliargs.verbose) { - display::print_analysis_results(cliargs.quiet, cliargs.extract, &results); - } - - // If running recursively, add extraction results to list of files to analyze - if cliargs.matryoshka { - for (_signature_id, extraction_result) in results.extractions.into_iter() { - if !extraction_result.do_not_recurse { - for file_path in extractors::common::get_extracted_files( - &extraction_result.output_directory, - ) { - debug!("Queuing {file_path} for analysis"); - target_files.insert(target_files.len(), file_path.clone()); - } - } - } - } - } - } - - json_logger.close(); - - // If BINWALK_RM_SYMLINK env var was set, delete the base_target_file symlink - if (cliargs.carve || cliargs.extract) && std::env::var(BINWALK_RM_SYMLINK).is_ok() { - if let Err(e) = std::fs::remove_file(&binwalker.base_target_file) { - error!( - "Request to remove extraction symlink file {} failed: {}", - binwalker.base_target_file, e - ); - } - } - - // All done, show some basic statistics - display::print_stats( - cliargs.quiet, - run_time, - file_count, - binwalker.signature_count, - binwalker.pattern_count, - ); - - ExitCode::SUCCESS -} - -/// Returns true if the specified results should be displayed to screen -fn should_display(results: &AnalysisResults, file_count: usize, verbose: bool) -> bool { - let mut display_results: bool = false; - - /* - * For brevity, when analyzing more than one file only display subsequent files whose results - * contain signatures that we always want displayed, or which contain extractable signatures. - * This can be overridden with the --verbose command line flag. - */ - if file_count == 1 || verbose || !results.extractions.is_empty() { - display_results = true; - } else { - for signature in &results.file_map { - if signature.always_display { - display_results = true; - break; - } - } - } - - display_results -} - -/// Spawn a worker thread to analyze a file -fn spawn_worker( - pool: &ThreadPool, - bw: binwalk::Binwalk, - target_file: String, - stdin: bool, - do_extraction: bool, - do_carve: bool, - worker_tx: mpsc::Sender, -) { - pool.execute(move || { - // Read in file data - let file_data = match common::read_input(&target_file, stdin) { - Err(_) => { - error!("Failed to read {target_file} data"); - b"".to_vec() - } - Ok(data) => data, - }; - - // Analyze target file, with extraction, if specified - let results = bw.analyze_buf(&file_data, &target_file, do_extraction); - - // If data carving was requested as part of extraction, carve analysis results to disk - if do_carve { - let carve_count = carve_file_map(&file_data, &results); - info!("Carved {carve_count} data blocks to disk from {target_file}"); - } - - // Report file results back to main thread - if let Err(e) = worker_tx.send(results) { - panic!( - "Worker thread for {target_file} failed to send results back to main thread: {e}" - ); - } - }); -} - -/// Carve signatures identified during analysis to separate files on disk. -/// Returns the number of carved files created. -/// Note that unknown blocks of file data are also carved to disk, so the number of files -/// created may be larger than the number of results defined in results.file_map. -fn carve_file_map(file_data: &[u8], results: &binwalk::AnalysisResults) -> usize { - let mut carve_count: usize = 0; - let mut last_known_offset: usize = 0; - let mut unknown_bytes: Vec<(usize, usize)> = Vec::new(); - - // No results, don't do anything - if !results.file_map.is_empty() { - // Loop through all identified signatures in the file - for signature_result in &results.file_map { - // If there is data between the last signature and this signature, it is some chunk of unknown data - if signature_result.offset > last_known_offset { - unknown_bytes.push(( - last_known_offset, - signature_result.offset - last_known_offset, - )); - } - - // Carve this signature's data to disk - if carve_file_data_to_disk( - &results.file_path, - file_data, - &signature_result.name, - signature_result.offset, - signature_result.size, - ) { - carve_count += 1; - } - - // Update the last known offset to the end of this signature's data - last_known_offset = signature_result.offset + signature_result.size; - } - - // Calculate the size of any remaining data from the end of the last signature to EOF - let remaining_data = file_data.len() - last_known_offset; - - // Add any remaining unknown data to the unknown_bytes list - if remaining_data > 0 { - unknown_bytes.push((last_known_offset, remaining_data)); - } - - // All known signature data has been carved to disk, now carve any unknown blocks of data to disk - for (offset, size) in unknown_bytes { - if carve_file_data_to_disk(&results.file_path, file_data, "unknown", offset, size) { - carve_count += 1; - } - } - } - - carve_count -} - -/// Carves a block of file data to a new file on disk -fn carve_file_data_to_disk( - source_file_path: &str, - file_data: &[u8], - name: &str, - offset: usize, - size: usize, -) -> bool { - let chroot = extractors::common::Chroot::new(None); - - // Carved file path will be: __.raw - let carved_file_path = format!("{source_file_path}_{offset}_{name}.raw",); - - debug!("Carving {carved_file_path}"); - - // Carve the data to disk - if !chroot.carve_file(&carved_file_path, file_data, offset, size) { - error!( - "Failed to carve {} [{:#X}..{:#X}] to disk", - carved_file_path, - offset, - offset + size, - ); - return false; - } - - true + let args = orchestrator::OrchestratorArgs::new_or_help(); + orchestrator::Orchestrator::new(args).scan() } diff --git a/src/orchestrator.rs b/src/orchestrator.rs new file mode 100644 index 000000000..30b90c378 --- /dev/null +++ b/src/orchestrator.rs @@ -0,0 +1,504 @@ +use std::{collections::VecDeque, process::ExitCode, time}; + +use clap::{CommandFactory, Parser}; + +use crate::{common, display, entropy, extractors, json, magic}; + +use crate::binwalk; +use crate::binwalk::AnalysisResults; +use log::{debug, error, info}; +use std::panic; +use std::process; +use std::sync::mpsc; +use std::thread; +use threadpool::ThreadPool; + +#[derive(Debug, Parser)] +#[command(author, version, about, long_about = None)] +pub struct OrchestratorArgs { + /// List supported signatures and extractors + #[arg(short = 'L', long)] + pub list: bool, + + /// Read data from standard input + #[arg(short, long)] + pub stdin: bool, + + /// Supress normal stdout output + #[arg(short, long)] + pub quiet: bool, + + /// During recursive extraction display *all* results + #[arg(short, long)] + pub verbose: bool, + + /// Automatically extract known file types + #[arg(short, long)] + pub extract: bool, + + /// Carve both known and unknown file contents to disk + #[arg(short, long)] + pub carve: bool, + + /// Recursively scan extracted files + #[arg(short = 'M', long)] + pub matryoshka: bool, + + /// Search for all signatures at all offsets + #[arg(short = 'a', long)] + pub search_all: bool, + + /// Generate an entropy graph with Plotly + #[arg(short = 'E', long, conflicts_with = "extract")] + pub entropy: bool, + + /// Save entropy graph as a PNG file + #[arg(short, long)] + pub png: Option, + + /// Log JSON results to a file ('-' for stdout) + #[arg(short, long)] + pub log: Option, + + /// Manually specify the number of threads to use + #[arg(short, long)] + pub threads: Option, + + /// Do no scan for these signatures + #[arg(short = 'x', long, value_delimiter = ',', num_args = 1..)] + pub exclude: Option>, + + /// Only scan for these signatures + #[arg(short = 'y', long, value_delimiter = ',', num_args = 1.., conflicts_with = "exclude")] + pub include: Option>, + + /// Extract files/folders to a custom directory + #[arg(short, long, default_value = "extractions")] + pub directory: String, + + /// Path to the file to analyze + pub file_name: Option, +} + +impl OrchestratorArgs { + pub fn new_or_help() -> OrchestratorArgs { + let args = OrchestratorArgs::parse(); + + if std::env::args().len() == 1 { + OrchestratorArgs::command() + .print_help() + .expect("Failed to print help output"); + std::process::exit(0); + } + + args + } +} + +pub struct Orchestrator { + pub args: OrchestratorArgs, +} + +impl Orchestrator { + pub fn new(args: OrchestratorArgs) -> Self { + Orchestrator { args } + } + + pub fn scan(self) -> ExitCode { + // File name used when reading from stdin + const STDIN: &str = "stdin"; + + // Only use one thread if unable to auto-detect available core info + const DEFAULT_WORKER_COUNT: usize = 1; + + // Number of seconds to wait before printing debug progress info + const PROGRESS_INTERVAL: u64 = 30; + + // If this env var is set during extraction, the Binwalk.base_target_file symlink will + // be deleted at the end of extraction. + const BINWALK_RM_SYMLINK: &str = "BINWALK_RM_EXTRACTION_SYMLINK"; + + // Destructure args into locals so fields can be moved freely without + // leaving `self` in a partially-moved state. + let OrchestratorArgs { + list, + stdin, + quiet, + verbose, + extract, + carve, + matryoshka, + search_all, + entropy: do_entropy, + png, + log, + threads, + exclude, + include, + directory, + mut file_name, + } = self.args; + + // Output directory for extracted files + let mut output_directory: Option = None; + + /* + * Maintain a queue of files waiting to be analyzed. + * Note that ThreadPool has its own internal queue so this may seem redundant, however, + * queuing a large number of jobs via the ThreadPool queue results in *massive* amounts + * of unecessary memory consumption, especially when recursively analyzing many files. + */ + let mut target_files = VecDeque::new(); + + // Statistics variables; keeps track of analyzed file count and total analysis run time + let mut file_count: usize = 0; + let run_time = time::Instant::now(); + let mut last_progress_interval = time::Instant::now(); + + // Initialize logging + env_logger::init(); + + // If --list was specified, just display a list of signatures and return + if list { + display::print_signature_list(quiet, &magic::patterns()); + return ExitCode::SUCCESS; + } + + // Set a dummy file name when reading from stdin + if stdin { + file_name = Some(STDIN.to_string()); + } + + let mut json_logger = json::JsonLogger::new(log); + + // If entropy analysis was requested, generate the entropy graph and return + if do_entropy { + display::print_plain(quiet, "Calculating file entropy..."); + + if let Ok(entropy_results) = entropy::plot(file_name.unwrap(), stdin, png) { + // Log entropy results to JSON file, if requested + json_logger.log(json::JSONType::Entropy(entropy_results.clone())); + json_logger.close(); + + display::println_plain(quiet, "done."); + } else { + panic!("Entropy analysis failed!"); + } + + return ExitCode::SUCCESS; + } + + // If extraction or data carving was requested, we need to initialize the output directory + if extract || carve { + output_directory = Some(directory); + } + + // Initialize binwalk + let binwalker = match binwalk::Binwalk::configure( + file_name, + output_directory, + include, + exclude, + None, + search_all, + ) { + Err(e) => { + error!("Binwalk initialization failed: {}", e.message); + return ExitCode::FAILURE; + } + Ok(bw) => bw, + }; + + // If the user specified --threads, honor that request; else, auto-detect available parallelism + let available_workers = threads.unwrap_or_else(|| { + // Get CPU core info + match thread::available_parallelism() { + // In case of error use the default + Err(e) => { + error!("Failed to retrieve CPU core info: {e}"); + DEFAULT_WORKER_COUNT + } + Ok(coreinfo) => coreinfo.get(), + } + }); + + // Sanity check the number of available worker threads + if available_workers < 1 { + panic!("No available worker threads!"); + } + + // Initialize thread pool + debug!("Initializing thread pool with {available_workers} workers"); + let workers = ThreadPool::new(available_workers); + let (worker_tx, worker_rx) = mpsc::channel(); + + /* + * Set a custom panic handler. + * This ensures that when any thread panics, the default panic handler will be invoked + * _and_ the entire process will exit with an error code. + */ + let default_panic_handler = panic::take_hook(); + panic::set_hook(Box::new(move |panic_info| { + default_panic_handler(panic_info); + process::exit(-1); + })); + + debug!( + "Queuing initial target file: {}", + binwalker.base_target_file + ); + + // Queue the initial file path + target_files.insert(target_files.len(), binwalker.base_target_file.clone()); + + /* + * Main loop. + * Loop until all pending thread jobs are complete and there are no more files in the queue. + */ + while !target_files.is_empty() || workers.active_count() > 0 { + // If there are files waiting to be analyzed and there is at least one free thread in the pool + if !target_files.is_empty() && workers.active_count() < workers.max_count() { + // Get the next file path from the target_files queue + let target_file = target_files + .pop_front() + .expect("Failed to retrieve next file from the queue"); + + // Spawn a new worker for the new file + Self::spawn_worker( + &workers, + binwalker.clone(), + target_file, + stdin && file_count == 0, + extract, + carve, + worker_tx.clone(), + ); + } + + // Don't spin CPU cycles if there is no backlog of files to analyze + if target_files.is_empty() { + let sleep_time = time::Duration::from_millis(1); + thread::sleep(sleep_time); + } + + // Some debug info on analysis progress + if last_progress_interval.elapsed().as_secs() >= PROGRESS_INTERVAL { + info!( + "Status: active worker threads: {}/{}, files waiting in queue: {}", + workers.active_count(), + workers.max_count(), + target_files.len() + ); + last_progress_interval = time::Instant::now(); + } + + // Get response from a worker thread, if any + if let Ok(results) = worker_rx.try_recv() { + // Keep a tally of how many files have been analyzed + file_count += 1; + + // Log analysis results to JSON file + json_logger.log(json::JSONType::Analysis(results.clone())); + + // Nothing found? Nothing else to do for this file. + if results.file_map.is_empty() { + debug!("Found no results for file {}", results.file_path); + continue; + } + + // Print analysis results to screen + if Self::should_display(&results, file_count, verbose) { + display::print_analysis_results(quiet, extract, &results); + } + + // If running recursively, add extraction results to list of files to analyze + if matryoshka { + for (_signature_id, extraction_result) in results.extractions.into_iter() { + if !extraction_result.do_not_recurse { + for file_path in extractors::common::get_extracted_files( + &extraction_result.output_directory, + ) { + debug!("Queuing {file_path} for analysis"); + target_files.insert(target_files.len(), file_path.clone()); + } + } + } + } + } + } + + json_logger.close(); + + // If BINWALK_RM_SYMLINK env var was set, delete the base_target_file symlink + if (carve || extract) && std::env::var(BINWALK_RM_SYMLINK).is_ok() { + if let Err(e) = std::fs::remove_file(&binwalker.base_target_file) { + error!( + "Request to remove extraction symlink file {} failed: {}", + binwalker.base_target_file, e + ); + } + } + + // All done, show some basic statistics + display::print_stats( + quiet, + run_time, + file_count, + binwalker.signature_count, + binwalker.pattern_count, + ); + + ExitCode::SUCCESS + } + + /// Returns true if the specified results should be displayed to screen + fn should_display(results: &AnalysisResults, file_count: usize, verbose: bool) -> bool { + let mut display_results: bool = false; + + /* + * For brevity, when analyzing more than one file only display subsequent files whose results + * contain signatures that we always want displayed, or which contain extractable signatures. + * This can be overridden with the --verbose command line flag. + */ + if file_count == 1 || verbose || !results.extractions.is_empty() { + display_results = true; + } else { + for signature in &results.file_map { + if signature.always_display { + display_results = true; + break; + } + } + } + + display_results + } + + /// Spawn a worker thread to analyze a file + fn spawn_worker( + pool: &ThreadPool, + bw: binwalk::Binwalk, + target_file: String, + stdin: bool, + do_extraction: bool, + do_carve: bool, + worker_tx: mpsc::Sender, + ) { + pool.execute(move || { + // Read in file data + let file_data = match common::read_input(&target_file, stdin) { + Err(_) => { + error!("Failed to read {target_file} data"); + b"".to_vec() + } + Ok(data) => data, + }; + + // Analyze target file, with extraction, if specified + let results = bw.analyze_buf(&file_data, &target_file, do_extraction); + + // If data carving was requested as part of extraction, carve analysis results to disk + if do_carve { + let carve_count = Self::carve_file_map(&file_data, &results); + info!("Carved {carve_count} data blocks to disk from {target_file}"); + } + + // Report file results back to main thread + if let Err(e) = worker_tx.send(results) { + panic!( + "Worker thread for {target_file} failed to send results back to main thread: {e}" + ); + } + }); + } + + /// Carve signatures identified during analysis to separate files on disk. + /// Returns the number of carved files created. + /// Note that unknown blocks of file data are also carved to disk, so the number of files + /// created may be larger than the number of results defined in results.file_map. + fn carve_file_map(file_data: &[u8], results: &binwalk::AnalysisResults) -> usize { + let mut carve_count: usize = 0; + let mut last_known_offset: usize = 0; + let mut unknown_bytes: Vec<(usize, usize)> = Vec::new(); + + // No results, don't do anything + if !results.file_map.is_empty() { + // Loop through all identified signatures in the file + for signature_result in &results.file_map { + // If there is data between the last signature and this signature, it is some chunk of unknown data + if signature_result.offset > last_known_offset { + unknown_bytes.push(( + last_known_offset, + signature_result.offset - last_known_offset, + )); + } + + // Carve this signature's data to disk + if Self::carve_file_data_to_disk( + &results.file_path, + file_data, + &signature_result.name, + signature_result.offset, + signature_result.size, + ) { + carve_count += 1; + } + + // Update the last known offset to the end of this signature's data + last_known_offset = signature_result.offset + signature_result.size; + } + + // Calculate the size of any remaining data from the end of the last signature to EOF + let remaining_data = file_data.len() - last_known_offset; + + // Add any remaining unknown data to the unknown_bytes list + if remaining_data > 0 { + unknown_bytes.push((last_known_offset, remaining_data)); + } + + // All known signature data has been carved to disk, now carve any unknown blocks of data to disk + for (offset, size) in unknown_bytes { + if Self::carve_file_data_to_disk( + &results.file_path, + file_data, + "unknown", + offset, + size, + ) { + carve_count += 1; + } + } + } + + carve_count + } + + /// Carves a block of file data to a new file on disk + fn carve_file_data_to_disk( + source_file_path: &str, + file_data: &[u8], + name: &str, + offset: usize, + size: usize, + ) -> bool { + let chroot = extractors::common::Chroot::new(None); + + // Carved file path will be: __.raw + let carved_file_path = format!("{source_file_path}_{offset}_{name}.raw",); + + debug!("Carving {carved_file_path}"); + + // Carve the data to disk + if !chroot.carve_file(&carved_file_path, file_data, offset, size) { + error!( + "Failed to carve {} [{:#X}..{:#X}] to disk", + carved_file_path, + offset, + offset + size, + ); + return false; + } + + true + } +}