From 5f9e7a9865d321cc9fae869d5ff5b4afb64a0941 Mon Sep 17 00:00:00 2001 From: Alan Szmyt Date: Mon, 31 Aug 2026 17:35:12 -0400 Subject: [PATCH] feat(toolchain): add runtime capability registry Introduce the canonical provider/tool registry, structured availability diagnostics, provider-aware planning, selected-tool fingerprints, CLI/SDK inspection surfaces, and generated registry documentation. Use the bounded process service for version probes, preserve provider identity through the transform graph, and make graph cache compatibility depend on the relevant selected toolchain rather than arbitrary host state. Closes #359 --- .github/workflows/docs.yml | 36 +- .../renderflow-core/data/tool-registry.yaml | 161 ++ .../renderflow-core/src/adapters/command.rs | 6 +- crates/renderflow-core/src/app.rs | 15 +- crates/renderflow-core/src/artifact/cache.rs | 11 +- crates/renderflow-core/src/artifact/store.rs | 9 +- crates/renderflow-core/src/cli.rs | 46 + crates/renderflow-core/src/commands/graph.rs | 27 +- .../src/commands/graph_build.rs | 34 +- crates/renderflow-core/src/commands/mod.rs | 1 + crates/renderflow-core/src/commands/plugin.rs | 19 +- crates/renderflow-core/src/commands/system.rs | 89 +- crates/renderflow-core/src/commands/tools.rs | 184 +++ crates/renderflow-core/src/deps.rs | 19 +- .../renderflow-core/src/graph/capability.rs | 9 + .../renderflow-core/src/graph/dag_executor.rs | 81 +- .../renderflow-core/src/graph/definition.rs | 4 + .../src/graph/execution_plan.rs | 38 + crates/renderflow-core/src/graph/mod.rs | 30 +- .../src/graph/renderers/text.rs | 11 +- .../src/graph/transform_edge.rs | 19 + crates/renderflow-core/src/lib.rs | 1 + crates/renderflow-core/src/process.rs | 103 +- crates/renderflow-core/src/sdk.rs | 78 +- crates/renderflow-core/src/strategies/pdf.rs | 33 +- crates/renderflow-core/src/toolchain.rs | 1362 +++++++++++++++++ .../src/transforms/aggregation.rs | 15 +- .../renderflow-core/src/transforms/command.rs | 2 +- .../src/transforms/yaml_loader.rs | 137 +- docs/cli-reference/tools.md | 37 + docs/user-guide/tool-registry.md | 59 + mkdocs.yml | 2 + scripts/generate_tool_registry_doc.py | 103 ++ 33 files changed, 2499 insertions(+), 282 deletions(-) create mode 100644 crates/renderflow-core/data/tool-registry.yaml create mode 100644 crates/renderflow-core/src/commands/tools.rs create mode 100644 crates/renderflow-core/src/toolchain.rs create mode 100644 docs/cli-reference/tools.md create mode 100644 docs/user-guide/tool-registry.md create mode 100755 scripts/generate_tool_registry_doc.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 09da200..db16782 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,10 +8,13 @@ on: - 'docs/**' - 'mkdocs.yml' - 'scripts/generate_supported_formats_doc.py' - - 'src/audio/format.rs' - - 'src/graph/format.rs' - - 'src/image/format.rs' - - 'src/input_format.rs' + - 'scripts/generate_tool_registry_doc.py' + - 'crates/renderflow-core/data/tool-registry.yaml' + - 'crates/renderflow-core/src/audio/format.rs' + - 'crates/renderflow-core/src/graph/format.rs' + - 'crates/renderflow-core/src/image/format.rs' + - 'crates/renderflow-core/src/input_format.rs' + - 'crates/renderflow-core/src/toolchain.rs' push: branches: - main @@ -23,10 +26,13 @@ on: - 'docs/**' - 'mkdocs.yml' - 'scripts/generate_supported_formats_doc.py' - - 'src/audio/format.rs' - - 'src/graph/format.rs' - - 'src/image/format.rs' - - 'src/input_format.rs' + - 'scripts/generate_tool_registry_doc.py' + - 'crates/renderflow-core/data/tool-registry.yaml' + - 'crates/renderflow-core/src/audio/format.rs' + - 'crates/renderflow-core/src/graph/format.rs' + - 'crates/renderflow-core/src/image/format.rs' + - 'crates/renderflow-core/src/input_format.rs' + - 'crates/renderflow-core/src/toolchain.rs' workflow_dispatch: permissions: @@ -55,13 +61,15 @@ jobs: - name: Install MkDocs Material run: | python -m pip install --upgrade pip - pip install mkdocs-material mike + pip install mkdocs-material mike pyyaml - name: Regenerate generated docs - run: python scripts/generate_supported_formats_doc.py + run: | + python scripts/generate_supported_formats_doc.py + python scripts/generate_tool_registry_doc.py - name: Verify generated docs are committed - run: git diff --exit-code -- docs/user-guide/supported-formats.md + run: git diff --exit-code -- docs/user-guide/supported-formats.md docs/user-guide/tool-registry.md - name: Build docs run: mkdocs build --strict @@ -91,10 +99,12 @@ jobs: - name: Install MkDocs Material run: | python -m pip install --upgrade pip - pip install mkdocs-material mike + pip install mkdocs-material mike pyyaml - name: Regenerate generated docs - run: python scripts/generate_supported_formats_doc.py + run: | + python scripts/generate_supported_formats_doc.py + python scripts/generate_tool_registry_doc.py - name: Configure Pages uses: actions/configure-pages@v5 diff --git a/crates/renderflow-core/data/tool-registry.yaml b/crates/renderflow-core/data/tool-registry.yaml new file mode 100644 index 0000000..110bc54 --- /dev/null +++ b/crates/renderflow-core/data/tool-registry.yaml @@ -0,0 +1,161 @@ +schema: renderflow.tool-registry/v1 + +tools: + - id: tool.pandoc + name: Pandoc + discovery: + kind: executable + candidates: [pandoc] + version_args: [--version] + version: + min_inclusive: "2.0.0" + operating_systems: [linux, macos, windows] + capabilities: + - document.convert + - document.generate + input_media_types: + - text/markdown + - text/html + - text/x-rst + - application/epub+zip + - application/x-tex + output_media_types: + - text/html + - application/pdf + - application/vnd.openxmlformats-officedocument.wordprocessingml.document + - application/epub+zip + - application/x-tex + determinism: configuration_dependent + locality: local + fidelity: path_dependent + support_tier: required + license_notes: "See the Pandoc upstream license for redistribution terms." + distribution_notes: "Typically installed through the host package manager or official Pandoc release artifacts." + + - id: tool.tectonic + name: Tectonic + discovery: + kind: executable + candidates: [tectonic] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - pdf.typeset + - latex.compile + input_media_types: + - application/x-tex + - text/x-tex + output_media_types: + - application/pdf + determinism: configuration_dependent + locality: network_optional + fidelity: path_dependent + support_tier: optional + license_notes: "See the Tectonic upstream license for redistribution terms." + distribution_notes: "Optional unless a selected PDF path uses Tectonic." + + - id: tool.ffmpeg + name: FFmpeg + discovery: + kind: executable + candidates: [ffmpeg] + version_args: [--version] + version: + min_inclusive: "4.0.0" + operating_systems: [linux, macos, windows] + capabilities: + - media.convert + - audio.convert + - image.convert + - video.convert + input_media_types: + - audio/* + - image/* + - video/* + output_media_types: + - audio/* + - image/* + - video/* + determinism: configuration_dependent + locality: local + fidelity: path_dependent + support_tier: optional + license_notes: "FFmpeg licensing is build-dependent; consult the exact distributed build." + distribution_notes: "Used by Renderflow audio, image, and future video adapters." + + - id: tool.wkhtmltopdf + name: wkhtmltopdf + discovery: + kind: executable + candidates: [wkhtmltopdf] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - html.render.pdf + input_media_types: [text/html] + output_media_types: [application/pdf] + determinism: configuration_dependent + locality: local + fidelity: partial_loss + support_tier: experimental + license_notes: "Consult upstream wkhtmltopdf licensing and bundled Qt terms." + distribution_notes: "Alternative HTML-to-PDF provider; not required by the default document pipeline." + + - id: tool.zip + name: Info-ZIP compatible zip + discovery: + kind: executable + candidates: [zip] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - archive.zip.create + - comic.cbz.create + output_media_types: + - application/zip + - application/vnd.comicbook+zip + determinism: configuration_dependent + locality: local + fidelity: lossless + support_tier: experimental + license_notes: "License depends on the compatible zip implementation selected on PATH." + distribution_notes: "Used by command-backed CBZ aggregation when configured." + + - id: tool.img2pdf + name: img2pdf + discovery: + kind: executable + candidates: [img2pdf] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - image.aggregate.pdf + input_media_types: [image/*] + output_media_types: [application/pdf] + determinism: deterministic + locality: local + fidelity: lossless + support_tier: experimental + license_notes: "Consult the img2pdf upstream license for redistribution terms." + distribution_notes: "Used by lossless image-to-PDF aggregation when configured." + + - id: tool.ghostscript + name: Ghostscript + discovery: + kind: executable + candidates: [gs] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - pdf.process + - tiff.aggregate.press_pdf + input_media_types: + - image/tiff + - application/pdf + output_media_types: [application/pdf] + determinism: configuration_dependent + locality: local + fidelity: path_dependent + support_tier: experimental + license_notes: "Ghostscript is available under AGPL/commercial licensing; review distribution obligations." + distribution_notes: "Used by press-oriented TIFF/PDF command adapters when configured." diff --git a/crates/renderflow-core/src/adapters/command.rs b/crates/renderflow-core/src/adapters/command.rs index 0fb2917..9ab6c6b 100644 --- a/crates/renderflow-core/src/adapters/command.rs +++ b/crates/renderflow-core/src/adapters/command.rs @@ -50,7 +50,11 @@ pub fn run_command(program: &str, args: &[&str]) -> Result<()> { } result.ensure_success()?; - info!(program = program, duration_ms = result.duration_ms(), "Command completed successfully"); + info!( + program = program, + duration_ms = result.duration_ms(), + "Command completed successfully" + ); Ok(()) } diff --git a/crates/renderflow-core/src/app.rs b/crates/renderflow-core/src/app.rs index 67b5487..f7cef06 100644 --- a/crates/renderflow-core/src/app.rs +++ b/crates/renderflow-core/src/app.rs @@ -2,7 +2,7 @@ use anyhow::{bail, Result}; use clap::Parser; use tracing::info; -use crate::cli::{AiCommands, Cli, Commands, GraphCommands, PluginCommands}; +use crate::cli::{AiCommands, Cli, Commands, GraphCommands, PluginCommands, ToolCommands}; use crate::{commands, transforms}; /// Initialize logging for a Renderflow CLI run. @@ -133,6 +133,19 @@ pub fn run_cli(cli: Cli) -> Result<()> { optimization, } => commands::graph::run_stats(&config, target.as_deref(), optimization)?, }, + Some(Commands::Tools { subcommand }) => match subcommand { + ToolCommands::List { format, transforms } => { + commands::tools::run_list(transforms.as_deref(), &format)? + } + ToolCommands::Inspect { + id, + format, + transforms, + } => commands::tools::run_inspect(&id, transforms.as_deref(), &format)?, + }, + Some(Commands::Capabilities { format, transforms }) => { + commands::tools::run_capabilities(transforms.as_deref(), &format)? + } Some(Commands::Version) => commands::system::run_version(), Some(Commands::Env) => commands::system::run_env(), Some(Commands::Doctor { strict }) => commands::system::run_doctor(strict)?, diff --git a/crates/renderflow-core/src/artifact/cache.rs b/crates/renderflow-core/src/artifact/cache.rs index 386e1c3..75d31c5 100644 --- a/crates/renderflow-core/src/artifact/cache.rs +++ b/crates/renderflow-core/src/artifact/cache.rs @@ -96,7 +96,9 @@ pub(crate) fn save_artifact_cache(cache: &ArtifactCache, path: &Path) -> Result< let mut temporary = tempfile::NamedTempFile::new_in(parent) .context("Failed to create artifact cache temporary file")?; serde_json::to_writer(&mut temporary, cache).context("Failed to serialize artifact cache")?; - temporary.flush().context("Failed to flush artifact cache")?; + temporary + .flush() + .context("Failed to flush artifact cache")?; temporary .as_file() .sync_all() @@ -104,7 +106,12 @@ pub(crate) fn save_artifact_cache(cache: &ArtifactCache, path: &Path) -> Result< temporary .persist(path) .map_err(|error| error.error) - .with_context(|| format!("Failed to atomically save artifact cache '{}'", path.display()))?; + .with_context(|| { + format!( + "Failed to atomically save artifact cache '{}'", + path.display() + ) + })?; Ok(()) } diff --git a/crates/renderflow-core/src/artifact/store.rs b/crates/renderflow-core/src/artifact/store.rs index 6a390d5..0ea0d3a 100644 --- a/crates/renderflow-core/src/artifact/store.rs +++ b/crates/renderflow-core/src/artifact/store.rs @@ -216,7 +216,9 @@ impl ArtifactStore { artifact.id() ) })?; - temporary.flush().context("Failed to flush final artifact")?; + temporary + .flush() + .context("Failed to flush final artifact")?; temporary .as_file() .sync_all() @@ -293,9 +295,8 @@ mod tests { fn identical_payloads_share_content_storage() { let directory = tempfile::tempdir().unwrap(); let store = ArtifactStore::new(directory.path()).unwrap(); - let descriptor = || { - ArtifactDescriptor::for_format(Format::Png, ArtifactStorageClass::Intermediate) - }; + let descriptor = + || ArtifactDescriptor::for_format(Format::Png, ArtifactStorageClass::Intermediate); let first = store.put_bytes(b"\x89PNG\x00", descriptor()).unwrap(); let second = store.put_bytes(b"\x89PNG\x00", descriptor()).unwrap(); assert_eq!(first.id(), second.id()); diff --git a/crates/renderflow-core/src/cli.rs b/crates/renderflow-core/src/cli.rs index 440d58c..76a676b 100644 --- a/crates/renderflow-core/src/cli.rs +++ b/crates/renderflow-core/src/cli.rs @@ -27,6 +27,9 @@ use crate::optimization::OptimizationMode; renderflow version Print the installed version\n \ renderflow env Print installation environment details\n \ renderflow doctor Run installation diagnostics\n \ + renderflow tools list List runtime tool providers\n \ + renderflow tools inspect tool.ffmpeg Inspect one runtime provider\n \ + renderflow capabilities List provider capability IDs\n \ renderflow my-project.yaml Shorthand: run build on the given config" )] pub struct Cli { @@ -198,6 +201,23 @@ pub enum Commands { subcommand: GraphCommands, }, + /// Inspect the runtime external-tool/provider registry. + #[command(subcommand_required = true, arg_required_else_help = true)] + Tools { + #[command(subcommand)] + subcommand: ToolCommands, + }, + + /// List stable provider capability IDs and their implementations. + Capabilities { + /// Output format: text (default), json, or yaml. + #[arg(long, default_value = "text", value_name = "FORMAT")] + format: String, + /// Optional transform YAML whose dynamic providers should be included. + #[arg(long, value_name = "FILE")] + transforms: Option, + }, + /// Print the installed Renderflow version Version, @@ -238,6 +258,32 @@ pub enum PluginCommands { Doctor, } +/// Subcommands for `renderflow tools`. +#[derive(Subcommand)] +pub enum ToolCommands { + /// List registered providers and live availability/version state. + List { + /// Output format: text (default), json, or yaml. + #[arg(long, default_value = "text", value_name = "FORMAT")] + format: String, + /// Optional transform YAML whose dynamic providers should be included. + #[arg(long, value_name = "FILE")] + transforms: Option, + }, + + /// Inspect one stable provider ID. + Inspect { + /// Stable provider/tool identifier, for example `tool.ffmpeg`. + id: String, + /// Output format: text (default), json, or yaml. + #[arg(long, default_value = "text", value_name = "FORMAT")] + format: String, + /// Optional transform YAML whose dynamic providers should be included. + #[arg(long, value_name = "FILE")] + transforms: Option, + }, +} + /// Subcommands for `renderflow ai`. #[derive(Subcommand)] pub enum AiCommands { diff --git a/crates/renderflow-core/src/commands/graph.rs b/crates/renderflow-core/src/commands/graph.rs index 447d4cc..c2fa6de 100644 --- a/crates/renderflow-core/src/commands/graph.rs +++ b/crates/renderflow-core/src/commands/graph.rs @@ -8,13 +8,14 @@ use crate::graph::execution_plan::ExecutionPlan; use crate::graph::renderers::renderer_for; use crate::graph::{Format, MultiTargetDag}; use crate::optimization::OptimizationMode; -use crate::transforms::yaml_loader::build_graph_and_executor_from_yaml; +use crate::toolchain::filter_graph_for_current_toolchain; +use crate::transforms::yaml_loader::build_graph_executor_and_tools_from_yaml; // ── helpers ───────────────────────────────────────────────────────────────── /// Load the config, build the transform graph, compute the DAG, and return an /// [`ExecutionPlan`]. -fn load_plan( +pub(crate) fn load_plan( config_path: &str, target: Option<&str>, optimization: Option, @@ -29,8 +30,14 @@ fn load_plan( ) })?; - let (graph, _executor) = build_graph_and_executor_from_yaml(transforms_path)?; - info!("Loaded transform graph from '{}'", transforms_path); + let (raw_graph, _executor, tool_registry) = + build_graph_executor_and_tools_from_yaml(transforms_path)?; + let (graph, tool_inventory, tool_context) = + filter_graph_for_current_toolchain(&raw_graph, &tool_registry); + info!( + "Loaded tool-aware transform graph from '{}'", + transforms_path + ); let opt_mode = optimization.unwrap_or(config.optimization); @@ -61,12 +68,18 @@ fn load_plan( .ok_or_else(|| { anyhow::anyhow!( "Could not build an execution plan: one or more target formats \ - are not reachable from '{}' in the transform graph", - source_format + are not reachable from '{}' after provider availability filtering. Blocked providers: {}", + source_format, + tool_inventory.blocked_summaries().join("; ") ) })?; - let plan = ExecutionPlan::from_dag(&dag, source_format, &targets, opt_mode); + let mut plan = ExecutionPlan::from_dag(&dag, source_format, &targets, opt_mode); + let toolchain = tool_registry.fingerprint_for_dag(&tool_inventory, &dag, &tool_context)?; + plan.attach_toolchain(toolchain); + for diagnostic in tool_inventory.blocked_summaries() { + plan.add_tool_diagnostic(format!("Provider excluded from planning: {diagnostic}")); + } Ok((plan, targets)) } diff --git a/crates/renderflow-core/src/commands/graph_build.rs b/crates/renderflow-core/src/commands/graph_build.rs index 14157df..b4ad2b9 100644 --- a/crates/renderflow-core/src/commands/graph_build.rs +++ b/crates/renderflow-core/src/commands/graph_build.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{fs, path::Path}; use anyhow::{Context, Result}; use tracing::{debug, info}; @@ -8,7 +8,8 @@ use crate::config::load_config_for_graph; use crate::files::ensure_output_dir; use crate::graph::Format; use crate::optimization::OptimizationMode; -use crate::transforms::yaml_loader::build_graph_and_executor_from_yaml; +use crate::toolchain::filter_graph_for_current_toolchain; +use crate::transforms::yaml_loader::build_graph_executor_and_tools_from_yaml; /// Run graph-based execution targeting a single output format. pub fn run_target( @@ -46,7 +47,7 @@ fn run_impl( optimization: Option, ) -> Result<()> { if dry_run { - info!("Dry-run mode enabled — no files will be created and no commands will be executed"); + info!("Dry-run mode enabled — no files or transform commands will be produced; bounded tool probes may run for planning"); } info!("Running graph-based build pipeline"); @@ -60,8 +61,14 @@ fn run_impl( ) })?; - let (graph, executor) = build_graph_and_executor_from_yaml(transforms_path)?; - info!("Loaded transform graph from '{}'", transforms_path); + let (raw_graph, executor, tool_registry) = + build_graph_executor_and_tools_from_yaml(transforms_path)?; + let (graph, tool_inventory, tool_context) = + filter_graph_for_current_toolchain(&raw_graph, &tool_registry); + info!( + "Loaded tool-aware transform graph from '{}'", + transforms_path + ); let opt_mode = optimization.unwrap_or(config.optimization); info!(optimization = %opt_mode, "Using optimization mode"); @@ -101,11 +108,15 @@ fn run_impl( .ok_or_else(|| { anyhow::anyhow!( "Could not build an execution plan: one or more target formats \ - are not reachable from '{}' in the transform graph", - source_format + are not reachable from '{}' after provider availability filtering. Blocked providers: {}", + source_format, + tool_inventory.blocked_summaries().join("; ") ) })?; + let toolchain = tool_registry.fingerprint_for_dag(&tool_inventory, &dag, &tool_context)?; + info!(fingerprint = %toolchain.fingerprint, providers = toolchain.selected_tools.len(), "Resolved execution toolchain"); + debug!("Execution plan (DAG tree):\n{}", dag.to_tree(source_format)); let input_stem = Path::new(&config.input) @@ -139,7 +150,14 @@ fn run_impl( .unwrap_or_else(|| Path::new(".")); let state_dir = state_parent.join(".renderflow"); let artifact_store = ArtifactStore::new(state_dir.join("artifacts"))?; - let executor = executor.with_cache(state_dir.join("dag-cache.json")); + fs::create_dir_all(&state_dir)?; + fs::write( + state_dir.join("toolchain.json"), + serde_json::to_vec_pretty(&toolchain)?, + )?; + let executor = executor + .with_cache(state_dir.join("dag-cache.json")) + .with_toolchain_fingerprint(toolchain.fingerprint.clone()); let source_artifact = artifact_store.import_path( &config.input, diff --git a/crates/renderflow-core/src/commands/mod.rs b/crates/renderflow-core/src/commands/mod.rs index f81d487..ba1e8a2 100644 --- a/crates/renderflow-core/src/commands/mod.rs +++ b/crates/renderflow-core/src/commands/mod.rs @@ -6,4 +6,5 @@ pub mod graph_build; pub mod inspect; pub mod plugin; pub mod system; +pub mod tools; pub mod watch; diff --git a/crates/renderflow-core/src/commands/plugin.rs b/crates/renderflow-core/src/commands/plugin.rs index 8cfc979..a75fff3 100644 --- a/crates/renderflow-core/src/commands/plugin.rs +++ b/crates/renderflow-core/src/commands/plugin.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use crate::process::ProcessExecutor; +use crate::toolchain::{CapabilityId, ToolRegistry}; use crate::transforms::plugin::{PluginCapabilities, PluginMetadata, PluginRegistry}; // ── list ────────────────────────────────────────────────────────────────────── @@ -173,7 +173,22 @@ pub fn run_doctor(registry: &PluginRegistry) -> Result<()> { /// Return `true` when `tool` can be found and successfully version-probed. fn tool_is_available(tool: &str) -> bool { - ProcessExecutor::new().probe_version(tool).is_available() + let mut registry = ToolRegistry::builtins(); + let id = registry.canonical_id_for_executable(tool); + if !registry.contains(&id) { + let capability = CapabilityId::new("plugin.required_tool") + .expect("static capability identifier is valid"); + if registry + .ensure_command_provider(id.clone(), tool.to_string(), capability) + .is_err() + { + return false; + } + } + registry + .assess_ids_current([id.as_str()]) + .get(id.as_str()) + .is_some_and(|availability| availability.is_available()) } #[cfg(test)] diff --git a/crates/renderflow-core/src/commands/system.rs b/crates/renderflow-core/src/commands/system.rs index 9ce4c87..aaebdd9 100644 --- a/crates/renderflow-core/src/commands/system.rs +++ b/crates/renderflow-core/src/commands/system.rs @@ -1,43 +1,7 @@ use anyhow::{bail, Result}; use std::{env, path::PathBuf}; -use crate::process::{ProcessExecutor, ToolProbeStatus}; - -struct ToolCheck { - name: &'static str, - required: bool, -} - -// `pandoc` is required for core document rendering, while `tectonic` (PDF) -// and `ffmpeg` (media conversions) are optional unless those outputs are used. -const TOOL_CHECKS: [ToolCheck; 3] = [ - ToolCheck { - name: "pandoc", - required: true, - }, - ToolCheck { - name: "tectonic", - required: false, - }, - ToolCheck { - name: "ffmpeg", - required: false, - }, -]; - -fn probe_tool_version(name: &str) -> Result { - let probe = ProcessExecutor::new().probe_version(name); - match probe.status { - ToolProbeStatus::Available => Ok(probe - .version_line - .unwrap_or_else(|| "available".to_string())), - ToolProbeStatus::Missing => Err(format!("missing ({name} not found in PATH)")), - ToolProbeStatus::TimedOut => Err(format!("installed but version probe timed out ({name} --version)")), - ToolProbeStatus::Failed => Err(probe - .diagnostic - .unwrap_or_else(|| format!("installed but failed to execute ({name} --version)"))), - } -} +use crate::toolchain::{ToolRegistry, ToolSupportTier}; pub fn run_version() { println!("renderflow {}", env!("CARGO_PKG_VERSION")); @@ -58,36 +22,49 @@ pub fn run_env() { } pub fn run_doctor(strict: bool) -> Result<()> { + let registry = ToolRegistry::builtins(); + let inventory = registry.assess_all_current(); + println!("Renderflow Doctor"); println!("-----------------"); println!("renderflow: {}", env!("CARGO_PKG_VERSION")); println!("platform: {} {}", env::consts::OS, env::consts::ARCH); + println!("tool registry: {}", registry.schema()); + println!(); - let mut missing = 0usize; - - for check in TOOL_CHECKS { - match probe_tool_version(check.name) { - Ok(version) => println!("[ok] {}: {version}", check.name), - Err(reason) => { - if check.required { - missing += 1; - println!("[missing|required] {}: {reason}", check.name); - } else { - println!("[missing|optional] {}: {reason}", check.name); - } - } + let mut required_failures = 0usize; + for tool in &inventory.tools { + if tool.support_tier == ToolSupportTier::Required && !tool.is_available() { + required_failures += 1; } + let detail = tool + .version_line + .as_deref() + .or(tool.diagnostic.as_deref()) + .unwrap_or("no additional detail"); + println!( + "[{}|{}] {}: {}", + tool.status.as_str(), + tool.support_tier.as_str(), + tool.id, + detail + ); } - if strict && missing > 0 { - bail!("doctor found {missing} required dependency issue(s)"); + if strict && required_failures > 0 { + bail!("doctor found {required_failures} required toolchain issue(s)"); } - if missing == 0 { - println!("Doctor completed: required dependencies look healthy."); + if required_failures == 0 { + println!( + " +Doctor completed: required toolchain providers look healthy." + ); } else { - println!("Doctor completed with warnings. Install missing required tools and retry."); + println!( + " +Doctor completed with required-provider warnings." + ); } - Ok(()) } diff --git a/crates/renderflow-core/src/commands/tools.rs b/crates/renderflow-core/src/commands/tools.rs new file mode 100644 index 0000000..c7bd510 --- /dev/null +++ b/crates/renderflow-core/src/commands/tools.rs @@ -0,0 +1,184 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, Result}; +use serde::Serialize; + +use crate::toolchain::{ToolAvailability, ToolDescriptor, ToolRegistry}; +use crate::transforms::yaml_loader::load_tool_registry_from_yaml; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StructuredFormat { + Text, + Json, + Yaml, +} + +impl StructuredFormat { + fn parse(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "text" => Ok(Self::Text), + "json" => Ok(Self::Json), + "yaml" | "yml" => Ok(Self::Yaml), + other => { + anyhow::bail!("unknown tools output format '{other}'; supported: text, json, yaml") + } + } + } +} + +fn load_registry(transforms: Option<&str>) -> Result { + match transforms { + Some(path) => load_tool_registry_from_yaml(path) + .with_context(|| format!("failed to load tool providers from '{path}'")), + None => Ok(ToolRegistry::builtins()), + } +} + +fn emit_serialized(value: &T, format: StructuredFormat) -> Result<()> { + match format { + StructuredFormat::Text => anyhow::bail!("text output requires a dedicated renderer"), + StructuredFormat::Json => { + println!("{}", serde_json::to_string_pretty(value)?); + } + StructuredFormat::Yaml => { + print!("{}", serde_yaml_ng::to_string(value)?); + } + } + Ok(()) +} + +pub fn run_list(transforms: Option<&str>, format: &str) -> Result<()> { + let registry = load_registry(transforms)?; + let inventory = registry.assess_all_current(); + let format = StructuredFormat::parse(format)?; + + if format != StructuredFormat::Text { + return emit_serialized(&inventory, format); + } + + println!("Renderflow Tools"); + println!("================"); + println!("registry: {}", registry.schema()); + println!(); + for tool in &inventory.tools { + let executable = tool.selected_executable.as_deref().unwrap_or("-"); + let version = tool + .normalized_version + .as_deref() + .or(tool.version_line.as_deref()) + .unwrap_or("-"); + println!( + "{:<24} {:<24} {:<12} {:<16} {}", + tool.id, + tool.status.as_str(), + tool.support_tier.as_str(), + executable, + version + ); + } + Ok(()) +} + +#[derive(Serialize)] +struct ToolInspection<'a> { + descriptor: &'a ToolDescriptor, + availability: &'a ToolAvailability, +} + +pub fn run_inspect(id: &str, transforms: Option<&str>, format: &str) -> Result<()> { + let registry = load_registry(transforms)?; + let descriptor = registry + .get(id) + .ok_or_else(|| anyhow::anyhow!("tool provider '{id}' is not registered"))?; + let inventory = registry.assess_ids_current([id]); + let availability = inventory + .get(id) + .expect("requested registered tool is present in its inventory"); + let format = StructuredFormat::parse(format)?; + + if format != StructuredFormat::Text { + return emit_serialized( + &ToolInspection { + descriptor, + availability, + }, + format, + ); + } + + println!("Tool: {}", descriptor.id); + println!("Name: {}", descriptor.name); + println!("Status: {}", availability.status.as_str()); + println!("Support tier: {}", descriptor.support_tier.as_str()); + println!( + "Executable: {}", + availability.selected_executable.as_deref().unwrap_or("-") + ); + println!( + "Version: {}", + availability + .version_line + .as_deref() + .unwrap_or("not captured") + ); + println!("Discovery: {:?}", descriptor.discovery); + println!("Determinism: {:?}", descriptor.determinism); + println!("Locality: {:?}", descriptor.locality); + println!("Fidelity: {:?}", descriptor.fidelity); + println!("Capabilities:"); + for capability in &descriptor.capabilities { + println!(" - {capability}"); + } + if !descriptor.fallbacks.is_empty() { + println!("Fallbacks:"); + for fallback in &descriptor.fallbacks { + println!(" - {fallback}"); + } + } + if let Some(diagnostic) = &availability.diagnostic { + println!("Diagnostic: {diagnostic}"); + } + Ok(()) +} + +pub fn run_capabilities(transforms: Option<&str>, format: &str) -> Result<()> { + let registry = load_registry(transforms)?; + let capabilities: BTreeMap> = registry.capabilities(); + let format = StructuredFormat::parse(format)?; + + if format != StructuredFormat::Text { + return emit_serialized(&capabilities, format); + } + + println!("Renderflow Capabilities"); + println!("======================="); + for (capability, providers) in capabilities { + println!("{capability}"); + for provider in providers { + println!(" - {provider}"); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn structured_format_rejects_unknown_value() { + assert!(StructuredFormat::parse("toml").is_err()); + } + + #[test] + fn built_in_capability_projection_is_not_empty() { + let registry = ToolRegistry::builtins(); + assert!(!registry.capabilities().is_empty()); + } + + #[test] + fn inventory_type_remains_structured_for_cli_serialization() { + let inventory = crate::toolchain::ToolInventory { tools: Vec::new() }; + assert!(serde_json::to_string(&inventory).is_ok()); + } +} diff --git a/crates/renderflow-core/src/deps.rs b/crates/renderflow-core/src/deps.rs index ba95f64..d12985e 100644 --- a/crates/renderflow-core/src/deps.rs +++ b/crates/renderflow-core/src/deps.rs @@ -1,12 +1,27 @@ use anyhow::Result; use crate::error::RenderError; -use crate::process::ProcessExecutor; +use crate::toolchain::{CapabilityId, ToolRegistry}; /// Check whether a tool is available in the system PATH using the canonical /// bounded version-probe path. fn tool_available(name: &str) -> bool { - ProcessExecutor::new().probe_version(name).is_available() + let mut registry = ToolRegistry::builtins(); + let id = registry.canonical_id_for_executable(name); + if !registry.contains(&id) { + let capability = + CapabilityId::new("probe.availability").expect("static capability identifier is valid"); + if registry + .ensure_command_provider(id.clone(), name.to_string(), capability) + .is_err() + { + return false; + } + } + registry + .assess_ids_current([id.as_str()]) + .get(id.as_str()) + .is_some_and(|tool| tool.is_available()) } /// Verify that `pandoc` is installed and available in PATH. diff --git a/crates/renderflow-core/src/graph/capability.rs b/crates/renderflow-core/src/graph/capability.rs index 20ce275..681acaa 100644 --- a/crates/renderflow-core/src/graph/capability.rs +++ b/crates/renderflow-core/src/graph/capability.rs @@ -183,6 +183,15 @@ impl ExternalTool { pub fn as_str(self) -> &'static str { self.executable_name() } + + /// Return the stable provider ID used by the runtime tool registry. + pub fn stable_id(self) -> &'static str { + match self { + Self::Pandoc => "tool.pandoc", + Self::Tectonic => "tool.tectonic", + Self::Ffmpeg => "tool.ffmpeg", + } + } } impl std::fmt::Display for ExternalTool { diff --git a/crates/renderflow-core/src/graph/dag_executor.rs b/crates/renderflow-core/src/graph/dag_executor.rs index d2d7fc5..6b9d847 100644 --- a/crates/renderflow-core/src/graph/dag_executor.rs +++ b/crates/renderflow-core/src/graph/dag_executor.rs @@ -8,9 +8,9 @@ use tracing::{debug, warn}; use super::{Format, MultiTargetDag, TransformEdge}; use crate::artifact::{ - compute_artifact_node_hash, load_artifact_cache, save_artifact_cache, Artifact, - ArtifactCache, ArtifactCollection, ArtifactDescriptor, ArtifactStorageClass, ArtifactStore, - ArtifactTransform, TextTransformAdapter, + compute_artifact_node_hash, load_artifact_cache, save_artifact_cache, Artifact, ArtifactCache, + ArtifactCollection, ArtifactDescriptor, ArtifactStorageClass, ArtifactStore, ArtifactTransform, + TextTransformAdapter, }; use crate::transforms::aggregation::AggregationTransform; use crate::transforms::Transform; @@ -27,6 +27,8 @@ pub struct DagExecutor { aggregation_transforms: HashMap<(Format, Format), Arc>, /// Optional artifact-native DAG cache path. cache_path: Option, + /// Selected-provider fingerprint used to reject incompatible cache entries. + toolchain_fingerprint: Option, } impl DagExecutor { @@ -36,6 +38,7 @@ impl DagExecutor { single_transforms: HashMap::new(), aggregation_transforms: HashMap::new(), cache_path: None, + toolchain_fingerprint: None, } } @@ -48,6 +51,12 @@ impl DagExecutor { self } + /// Attach a selected-provider fingerprint to cache compatibility. + pub fn with_toolchain_fingerprint(mut self, fingerprint: impl Into) -> Self { + self.toolchain_fingerprint = Some(fingerprint.into()); + self + } + /// Register an existing UTF-8 text transform through the compatibility adapter. pub fn register_single( &mut self, @@ -55,10 +64,8 @@ impl DagExecutor { to: Format, transform: Arc, ) -> &mut Self { - self.single_transforms.insert( - (from, to), - Arc::new(TextTransformAdapter::new(transform)), - ); + self.single_transforms + .insert((from, to), Arc::new(TextTransformAdapter::new(transform))); self } @@ -302,9 +309,12 @@ impl DagExecutor { edge.to ) })?; - let cache_identity = transform.cache_identity(); - let cache_key = - compute_artifact_node_hash(input, edge.from, edge.to, &cache_identity); + let mut cache_identity = transform.cache_identity(); + if let Some(fingerprint) = &self.toolchain_fingerprint { + cache_identity.push_str("\0toolchain="); + cache_identity.push_str(fingerprint); + } + let cache_key = compute_artifact_node_hash(input, edge.from, edge.to, &cache_identity); if let Some(cache_mutex) = cache { if let Ok(guard) = cache_mutex.lock() { @@ -490,11 +500,8 @@ mod tests { let mut reader = store.open(input)?; store.put_reader( &mut reader, - ArtifactDescriptor::for_format( - output_format, - ArtifactStorageClass::Intermediate, - ) - .with_source(input.id().clone()), + ArtifactDescriptor::for_format(output_format, ArtifactStorageClass::Intermediate) + .with_source(input.id().clone()), ) } } @@ -522,11 +529,8 @@ mod tests { let mut reader = store.open(input)?; store.put_reader( &mut reader, - ArtifactDescriptor::for_format( - output_format, - ArtifactStorageClass::Intermediate, - ) - .with_source(input.id().clone()), + ArtifactDescriptor::for_format(output_format, ArtifactStorageClass::Intermediate) + .with_source(input.id().clone()), ) } } @@ -563,11 +567,7 @@ mod tests { fn one_edge(from: Format, to: Format, input_kind: InputKind) -> MultiTargetDag { let mut graph = TransformGraph::new(); graph.add_transform(TransformEdge::with_input_kind( - from, - to, - 1.0, - 1.0, - input_kind, + from, to, 1.0, 1.0, input_kind, )); graph .build_multi_target_dag(from, &[to]) @@ -602,11 +602,7 @@ mod tests { ) .unwrap(); let mut executor = DagExecutor::new(); - executor.register_artifact( - Format::Png, - Format::Webp, - Arc::new(BinaryCopyTransform), - ); + executor.register_artifact(Format::Png, Format::Webp, Arc::new(BinaryCopyTransform)); let results = executor .execute_artifact(&dag, Format::Png, source.clone(), &store) @@ -635,11 +631,7 @@ mod tests { ) .unwrap(); let mut executor = DagExecutor::new(); - executor.register_aggregation( - Format::Png, - Format::Pdf, - Arc::new(OrderedJoinAggregation), - ); + executor.register_aggregation(Format::Png, Format::Pdf, Arc::new(OrderedJoinAggregation)); let results = executor .execute_artifacts( @@ -651,10 +643,7 @@ mod tests { .unwrap(); let output = results[&Format::Pdf].clone().into_one().unwrap(); assert_eq!(store.read_bytes(&output).unwrap(), b"page-one|page-two"); - assert_eq!( - output.sources(), - &[first.id().clone(), second.id().clone()] - ); + assert_eq!(output.sources(), &[first.id().clone(), second.id().clone()]); } #[test] @@ -700,19 +689,11 @@ mod tests { ) .unwrap(); let mut executor = DagExecutor::new(); - executor.register_aggregation( - Format::Png, - Format::Pdf, - Arc::new(FailingAggregation), - ); + executor.register_aggregation(Format::Png, Format::Pdf, Arc::new(FailingAggregation)); let before = count_artifact_objects(&store); - let result = executor.execute_artifacts( - &dag, - Format::Png, - ArtifactCollection::one(source), - &store, - ); + let result = + executor.execute_artifacts(&dag, Format::Png, ArtifactCollection::one(source), &store); assert!(result.is_err()); assert_eq!(before, count_artifact_objects(&store)); } diff --git a/crates/renderflow-core/src/graph/definition.rs b/crates/renderflow-core/src/graph/definition.rs index ab5ce21..b4f56d4 100644 --- a/crates/renderflow-core/src/graph/definition.rs +++ b/crates/renderflow-core/src/graph/definition.rs @@ -1,4 +1,5 @@ use super::{Format, InputKind, TransformEdge}; +use crate::toolchain::{canonical_tool_id_for_hint, transform_capability_id}; /// A pluggable definition of a format-to-format transformation. /// @@ -103,7 +104,10 @@ impl TransformDefinition { /// Convert this definition into a [`TransformEdge`] for use in a /// [`TransformGraph`](super::TransformGraph). pub fn to_edge(&self) -> TransformEdge { + let provider = canonical_tool_id_for_hint(&self.label); + let capability = transform_capability_id(self.from, self.to); TransformEdge::with_input_kind(self.from, self.to, self.cost, self.quality, self.input_kind) + .with_provider(provider.to_string(), capability.to_string()) } } diff --git a/crates/renderflow-core/src/graph/execution_plan.rs b/crates/renderflow-core/src/graph/execution_plan.rs index c478ece..8938306 100644 --- a/crates/renderflow-core/src/graph/execution_plan.rs +++ b/crates/renderflow-core/src/graph/execution_plan.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use super::{Format, InputKind, MultiTargetDag, TransformEdge}; use crate::optimization::OptimizationMode; +use crate::toolchain::ToolchainSnapshot; // ── Node types ───────────────────────────────────────────────────────────── @@ -60,6 +61,12 @@ pub struct PlanEdge { pub input_kind: String, /// Semantic classification of this edge. pub edge_type: EdgeType, + /// Stable provider/tool identifier selected for this edge. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_id: Option, + /// Stable capability identifier implemented by the selected provider. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capability_id: Option, } impl PlanEdge { @@ -82,6 +89,8 @@ impl PlanEdge { InputKind::Collection => "collection".to_string(), }, edge_type, + provider_id: e.provider_id.clone(), + capability_id: e.capability_id.clone(), } } } @@ -206,6 +215,9 @@ pub struct ExecutionPlan { pub metadata: ExecutionMetadata, /// Planner observations and explanations. pub diagnostics: Vec, + /// Reproducible evidence for providers selected by this exact plan. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub toolchain: Option, } impl ExecutionPlan { @@ -313,9 +325,35 @@ impl ExecutionPlan { waves, metadata, diagnostics, + toolchain: None, } } + /// Attach selected-provider evidence and a deterministic toolchain fingerprint. + pub fn attach_toolchain(&mut self, snapshot: ToolchainSnapshot) { + let selected = snapshot + .selected_tools + .iter() + .map(|tool| tool.id.to_string()) + .collect::>() + .join(", "); + self.diagnostics.push(PlanDiagnostic::info(format!( + "Toolchain fingerprint {} selected provider(s): {}.", + snapshot.fingerprint, + if selected.is_empty() { + "(none)" + } else { + &selected + } + ))); + self.toolchain = Some(snapshot); + } + + /// Surface an unavailable/unsupported provider observation in plan diagnostics. + pub fn add_tool_diagnostic(&mut self, message: impl Into) { + self.diagnostics.push(PlanDiagnostic::warning(message)); + } + // ── private helpers ──────────────────────────────────────────────────── /// Group edges into parallel execution waves. diff --git a/crates/renderflow-core/src/graph/mod.rs b/crates/renderflow-core/src/graph/mod.rs index 5308a47..4a30ec5 100644 --- a/crates/renderflow-core/src/graph/mod.rs +++ b/crates/renderflow-core/src/graph/mod.rs @@ -24,7 +24,7 @@ pub use transform_edge::TransformEdge; use crate::optimization::OptimizationMode; use petgraph::graph::{DiGraph, NodeIndex}; use petgraph::visit::EdgeRef; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// A directed graph of document format transformations. /// @@ -123,6 +123,34 @@ impl TransformGraph { .collect() } + /// Return stable provider IDs referenced by graph edges. + pub fn provider_ids(&self) -> Vec { + let mut ids: Vec = self + .graph + .edge_references() + .filter_map(|edge| edge.weight().provider_id.clone()) + .collect(); + ids.sort(); + ids.dedup(); + ids + } + + /// Clone the graph while excluding edges whose declared provider is unavailable. + /// Legacy edges without provider identity remain available for compatibility. + pub fn filtered_by_available_providers(&self, available: &HashSet) -> Self { + let mut filtered = Self::new(); + for edge in self.graph.edge_references().map(|edge| edge.weight()) { + if edge + .provider_id + .as_ref() + .is_none_or(|provider| available.contains(provider)) + { + filtered.add_transform(edge.clone()); + } + } + filtered + } + /// Return `true` when at least one direct transformation from `from` to /// `to` has been registered. pub fn has_transform(&self, from: Format, to: Format) -> bool { diff --git a/crates/renderflow-core/src/graph/renderers/text.rs b/crates/renderflow-core/src/graph/renderers/text.rs index 0f064a8..5bc7a5e 100644 --- a/crates/renderflow-core/src/graph/renderers/text.rs +++ b/crates/renderflow-core/src/graph/renderers/text.rs @@ -18,6 +18,9 @@ impl PlanRenderer for TextRenderer { let _ = writeln!(out, "Source: {}", plan.source); let _ = writeln!(out, "Targets: {}", plan.targets.join(", ")); let _ = writeln!(out, "Optimization: {}", plan.optimization); + if let Some(toolchain) = &plan.toolchain { + let _ = writeln!(out, "Toolchain: {}", toolchain.fingerprint); + } // ── nodes ────────────────────────────────────────────────────────── let _ = writeln!(out); @@ -36,13 +39,19 @@ impl PlanRenderer for TextRenderer { let _ = writeln!(out, "Edges ({}):", plan.metadata.total_edges); let max_from = plan.edges.iter().map(|e| e.from.len()).max().unwrap_or(0); for e in &plan.edges { + let provider = e + .provider_id + .as_deref() + .map(|value| format!(", provider: {value}")) + .unwrap_or_default(); let _ = writeln!( out, - " {:, + /// Stable machine-readable capability identifier for this edge, when known. + pub capability_id: Option, } impl TransformEdge { @@ -41,6 +45,8 @@ impl TransformEdge { cost, quality: quality.clamp(0.0, 1.0), input_kind: InputKind::Single, + provider_id: None, + capability_id: None, } } @@ -68,8 +74,21 @@ impl TransformEdge { cost, quality: quality.clamp(0.0, 1.0), input_kind, + provider_id: None, + capability_id: None, } } + + /// Attach stable provider and capability identity used by planning/evidence. + pub fn with_provider( + mut self, + provider_id: impl Into, + capability_id: impl Into, + ) -> Self { + self.provider_id = Some(provider_id.into()); + self.capability_id = Some(capability_id.into()); + self + } } #[cfg(test)] diff --git a/crates/renderflow-core/src/lib.rs b/crates/renderflow-core/src/lib.rs index 1228140..8884b8b 100644 --- a/crates/renderflow-core/src/lib.rs +++ b/crates/renderflow-core/src/lib.rs @@ -28,6 +28,7 @@ pub mod process; mod sdk; pub mod strategies; mod template; +pub mod toolchain; pub mod transforms; pub use sdk::{ diff --git a/crates/renderflow-core/src/process.rs b/crates/renderflow-core/src/process.rs index 74ba329..c0aad28 100644 --- a/crates/renderflow-core/src/process.rs +++ b/crates/renderflow-core/src/process.rs @@ -124,10 +124,7 @@ impl fmt::Debug for ProcessInput { match self { Self::Null => f.write_str("Null"), Self::Inherit => f.write_str("Inherit"), - Self::Bytes(bytes) => f - .debug_struct("Bytes") - .field("len", &bytes.len()) - .finish(), + Self::Bytes(bytes) => f.debug_struct("Bytes").field("len", &bytes.len()).finish(), } } } @@ -237,7 +234,8 @@ impl ProcessEnvironment { } pub fn allow_sensitive(mut self, name: impl Into) -> Self { - self.allow_sensitive.insert(normalize_env_name(&name.into())); + self.allow_sensitive + .insert(normalize_env_name(&name.into())); self } @@ -259,11 +257,7 @@ impl ProcessEnvironment { self } - pub fn set_sensitive( - mut self, - name: impl Into, - value: impl Into, - ) -> Self { + pub fn set_sensitive(mut self, name: impl Into, value: impl Into) -> Self { let name = name.into(); self.overrides.insert( normalize_env_name(&name), @@ -389,7 +383,10 @@ impl fmt::Debug for ProcessRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ProcessRequest") .field("executable", &self.executable) - .field("args", &safe_argument_display(&self.args, &Redactor::default())) + .field( + "args", + &safe_argument_display(&self.args, &Redactor::default()), + ) .field("invocation_kind", &self.invocation_kind) .field("working_directory", &self.working_directory) .field("stdin", &self.stdin) @@ -451,8 +448,11 @@ impl ProcessRequest { I: IntoIterator, S: Into, { - self.args - .extend(values.into_iter().map(|value| ProcessArgument::plain(value))); + self.args.extend( + values + .into_iter() + .map(|value| ProcessArgument::plain(value)), + ); self } @@ -682,26 +682,29 @@ impl ProcessExecutor { .is_some_and(ProcessCancellationToken::is_cancelled) { terminate_process_tree(&mut child, request.tree_mode).map_err(|error| { - ProcessError::Io(redactor.redact(&format!( - "failed to terminate cancelled process: {error}" - ))) + ProcessError::Io( + redactor.redact(&format!("failed to terminate cancelled process: {error}")), + ) })?; break ProcessTermination::Cancelled; } - if request.timeout.is_some_and(|timeout| started.elapsed() >= timeout) { + if request + .timeout + .is_some_and(|timeout| started.elapsed() >= timeout) + { terminate_process_tree(&mut child, request.tree_mode).map_err(|error| { - ProcessError::Io(redactor.redact(&format!( - "failed to terminate timed-out process: {error}" - ))) + ProcessError::Io( + redactor.redact(&format!("failed to terminate timed-out process: {error}")), + ) })?; break ProcessTermination::TimedOut; } match child.try_wait().map_err(|error| { - ProcessError::Io(redactor.redact(&format!( - "failed while waiting for process: {error}" - ))) + ProcessError::Io( + redactor.redact(&format!("failed while waiting for process: {error}")), + ) })? { Some(status) => break termination_from_status(status), None => thread::sleep(POLL_INTERVAL), @@ -715,9 +718,9 @@ impl ProcessExecutor { debug!(error = %redactor.redact(&error.to_string()), "stdin writer ended after process termination"); } Ok(Err(error)) => { - return Err(ProcessError::Io(redactor.redact(&format!( - "failed to write process stdin: {error}" - )))); + return Err(ProcessError::Io( + redactor.redact(&format!("failed to write process stdin: {error}")), + )); } Err(_) if !termination.is_success() => {} Err(_) => { @@ -777,8 +780,17 @@ impl ProcessExecutor { /// Probe ` --version` using the same bounded process policy. pub fn probe_version(&self, executable: &str) -> ToolProbeEvidence { + self.probe_version_with_args(executable, &["--version".to_string()]) + } + + /// Probe a tool version using an explicit argv declaration from the tool registry. + pub fn probe_version_with_args( + &self, + executable: &str, + version_args: &[String], + ) -> ToolProbeEvidence { let request = ProcessRequest::direct(executable) - .arg("--version") + .args(version_args.iter().cloned()) .timeout(PROBE_TIMEOUT) .capture_limit(PROBE_CAPTURE_LIMIT_BYTES); @@ -882,7 +894,8 @@ impl CapturedOutput { if self.truncated { format!( "{text}\n[capture truncated: retained {} of {} bytes]", - self.bytes.len(), self.total_bytes + self.bytes.len(), + self.total_bytes ) } else { text.to_string() @@ -960,7 +973,10 @@ impl ProcessResult { format!("Command `{}` exited with code {code}", self.command_display) } ProcessTermination::Signaled => { - format!("Command `{}` was terminated by a signal", self.command_display) + format!( + "Command `{}` was terminated by a signal", + self.command_display + ) } ProcessTermination::TimedOut => { format!("Command `{}` timed out", self.command_display) @@ -1093,12 +1109,18 @@ impl ProcessExpectedOutput { fn validate(&self, before: &OutputSnapshot) -> Option { let after = OutputSnapshot::capture(&self.path); if !after.exists { - return Some(format!("expected output '{}' was not produced", self.path.display())); + return Some(format!( + "expected output '{}' was not produced", + self.path.display() + )); } match self.kind { ExpectedOutputKind::Any => {} ExpectedOutputKind::File if !after.is_file => { - return Some(format!("expected output '{}' is not a file", self.path.display())); + return Some(format!( + "expected output '{}' is not a file", + self.path.display() + )); } ExpectedOutputKind::Directory if !after.is_directory => { return Some(format!( @@ -1109,7 +1131,10 @@ impl ProcessExpectedOutput { _ => {} } if self.require_non_empty && after.is_file && after.len == Some(0) { - return Some(format!("expected output '{}' is empty", self.path.display())); + return Some(format!( + "expected output '{}' is empty", + self.path.display() + )); } if self.require_change && &after == before { return Some(format!( @@ -1139,7 +1164,8 @@ impl Redactor { { self.secrets .extend(values.into_iter().filter(|value| !value.is_empty())); - self.secrets.sort_by_key(|value| std::cmp::Reverse(value.len())); + self.secrets + .sort_by_key(|value| std::cmp::Reverse(value.len())); self.secrets.dedup(); } @@ -1159,7 +1185,10 @@ struct BoundedBytes { truncated: bool, } -fn spawn_bounded_reader(reader: R, max_bytes: usize) -> thread::JoinHandle> +fn spawn_bounded_reader( + reader: R, + max_bytes: usize, +) -> thread::JoinHandle> where R: Read + Send + 'static, { @@ -1351,8 +1380,7 @@ pub(crate) fn is_explicit_shell_invocation(executable: &str, args: &[String]) -> .to_ascii_lowercase(); let is_shell = matches!( basename.as_str(), - "sh" - | "bash" + "sh" | "bash" | "zsh" | "dash" | "ksh" @@ -1542,8 +1570,7 @@ mod tests { let output = directory.path().join("missing.out"); let error = ProcessExecutor::new() .execute_checked( - ProcessRequest::direct("true") - .expect_output(ProcessExpectedOutput::file(&output)), + ProcessRequest::direct("true").expect_output(ProcessExpectedOutput::file(&output)), ) .unwrap_err(); assert!(error.to_string().contains("was not produced")); diff --git a/crates/renderflow-core/src/sdk.rs b/crates/renderflow-core/src/sdk.rs index f68ec39..90f0bd3 100644 --- a/crates/renderflow-core/src/sdk.rs +++ b/crates/renderflow-core/src/sdk.rs @@ -8,10 +8,10 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::commands; -use crate::config::{load_config, load_config_for_graph}; -use crate::graph::{ExecutionPlan, Format}; +use crate::config::load_config; +use crate::graph::ExecutionPlan; use crate::optimization::OptimizationMode; -use crate::transforms::yaml_loader::build_graph_and_executor_from_yaml; +use crate::toolchain::ToolchainSnapshot; #[derive(Debug, Error)] pub enum RenderflowError { @@ -172,6 +172,8 @@ pub struct ExecutionResult { pub reused_cached_outputs: Vec, pub skipped_transforms: Vec, pub diagnostics: DiagnosticReport, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub toolchain: Option, } #[derive(Default)] @@ -262,58 +264,17 @@ impl Engine { pub fn plan(&self, request: PlanRequest) -> Result { self.ensure_not_cancelled()?; self.emit(ProgressStage::Planning, "Constructing execution plan"); - - let config = load_config_for_graph(request.config_path.to_str().ok_or_else(|| { + let config_path = request.config_path.to_str().ok_or_else(|| { RenderflowError::Planning(anyhow::anyhow!("Config path contains non-UTF8 characters")) - })?) - .map_err(RenderflowError::Planning)?; - - let transforms_path = config.transforms.as_deref().ok_or_else(|| { - RenderflowError::Planning(anyhow::anyhow!( - "Planning requires a `transforms` key in the config file" - )) })?; - - let (graph, _executor) = build_graph_and_executor_from_yaml(transforms_path) - .map_err(RenderflowError::Planning)?; - - let opt_mode = request.optimization.unwrap_or(config.optimization); - let source_format: Format = config - .input_format() - .to_string() - .parse() - .map_err(|err| RenderflowError::Planning(anyhow::anyhow!("{}", err)))?; - - let targets: Vec = if let Some(target) = request.target { - vec![target - .parse::() - .map_err(|err| RenderflowError::Planning(anyhow::anyhow!("{}", err)))?] - } else { - let reachable = graph.reachable_from(source_format); - if reachable.is_empty() { - return Err(RenderflowError::Planning(anyhow::anyhow!( - "No output formats are reachable from '{}'", - source_format - ))); - } - reachable - }; - - let dag = graph - .build_multi_target_dag_with_mode(source_format, &targets, opt_mode) - .ok_or_else(|| { - RenderflowError::Planning(anyhow::anyhow!( - "Could not build an execution plan for one or more targets" - )) - })?; - + let (plan, _targets) = commands::graph::load_plan( + config_path, + request.target.as_deref(), + request.optimization, + ) + .map_err(RenderflowError::Planning)?; self.emit(ProgressStage::Completed, "Planning complete"); - Ok(ExecutionPlan::from_dag( - &dag, - source_format, - &targets, - opt_mode, - )) + Ok(plan) } pub fn execute(&self, request: ExecutionRequest) -> Result { @@ -326,6 +287,18 @@ impl Engine { let config = load_config(config_path).map_err(RenderflowError::Execution)?; + let toolchain = if request.target.is_some() || request.all_targets { + let (plan, _targets) = commands::graph::load_plan( + config_path, + request.target.as_deref(), + request.optimization, + ) + .map_err(RenderflowError::Execution)?; + plan.toolchain + } else { + None + }; + if let Some(target) = request.target.as_deref() { commands::graph_build::run_target( config_path, @@ -365,6 +338,7 @@ impl Engine { warnings: Vec::new(), recoverable_failures: Vec::new(), }, + toolchain, }) } } diff --git a/crates/renderflow-core/src/strategies/pdf.rs b/crates/renderflow-core/src/strategies/pdf.rs index 4807e9e..394c330 100644 --- a/crates/renderflow-core/src/strategies/pdf.rs +++ b/crates/renderflow-core/src/strategies/pdf.rs @@ -3,8 +3,8 @@ use std::path::Path; use tracing::info; use crate::adapters::command::run_command; -use crate::process::{ProcessExecutor, ToolProbeStatus}; use crate::strategies::{OutputStrategy, PandocArgs, RenderContext}; +use crate::toolchain::ToolRegistry; /// Renders a document to PDF format using pandoc with the tectonic PDF engine. pub struct PdfStrategy { @@ -23,26 +23,20 @@ impl PdfStrategy { /// Returns an error if the tectonic PDF engine is not installed or cannot /// be version-probed through the canonical process executor. fn check_tectonic() -> Result<()> { - let probe = ProcessExecutor::new().probe_version("tectonic"); - match probe.status { - ToolProbeStatus::Available => Ok(()), - ToolProbeStatus::Missing => anyhow::bail!( - "PDF rendering failed: `tectonic` is not installed.\n\n\ + let registry = ToolRegistry::builtins(); + let inventory = registry.assess_ids_current(["tool.tectonic"]); + if !inventory + .get("tool.tectonic") + .is_some_and(|tool| tool.is_available()) + { + anyhow::bail!( + "PDF rendering failed: `tectonic` is not installed or is incompatible.\n\n\ Fix:\n\ - Install tectonic: https://tectonic-typesetting.github.io/en-US/\n\ - Or configure a different PDF engine" - ), - ToolProbeStatus::TimedOut => anyhow::bail!( - "PDF rendering failed: `tectonic --version` timed out. \ - Verify the tectonic installation before retrying." - ), - ToolProbeStatus::Failed => anyhow::bail!( - "PDF rendering failed: tectonic is installed but its version probe failed: {}", - probe - .diagnostic - .unwrap_or_else(|| "unknown process failure".to_string()) - ), + ); } + Ok(()) } } @@ -122,7 +116,10 @@ mod tests { } fn tectonic_available() -> bool { - ProcessExecutor::new().probe_version("tectonic").is_available() + ToolRegistry::builtins() + .assess_ids_current(["tool.tectonic"]) + .get("tool.tectonic") + .is_some_and(|tool| tool.is_available()) } #[test] diff --git a/crates/renderflow-core/src/toolchain.rs b/crates/renderflow-core/src/toolchain.rs new file mode 100644 index 0000000..563f2e8 --- /dev/null +++ b/crates/renderflow-core/src/toolchain.rs @@ -0,0 +1,1362 @@ +//! Runtime external-tool capability registry and reproducible toolchain evidence. +//! +//! The registry separates tool identity/capability declarations from live host +//! probing. Built-in descriptors are loaded from `data/tool-registry.yaml`, while +//! plugins and YAML command transforms can register additional descriptors at +//! runtime through the same public API. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::env; +use std::fmt; +use std::str::FromStr; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::graph::{Format, MultiTargetDag, TransformGraph}; +use crate::process::{ProcessExecutor, ToolProbeEvidence, ToolProbeStatus as ProcessProbeStatus}; + +pub const TOOL_REGISTRY_SCHEMA: &str = "renderflow.tool-registry/v1"; +pub const TOOLCHAIN_SCHEMA: &str = "renderflow.toolchain/v1"; + +/// Stable machine-readable tool/provider identifier. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ToolId(String); + +impl ToolId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + validate_identifier("tool id", &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for ToolId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl FromStr for ToolId { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + Self::new(value) + } +} + +/// Stable machine-readable capability identifier. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct CapabilityId(String); + +impl CapabilityId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + validate_identifier("capability id", &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for CapabilityId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl FromStr for CapabilityId { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + Self::new(value) + } +} + +fn validate_identifier(kind: &str, value: &str) -> Result<()> { + if value.is_empty() { + anyhow::bail!("{kind} must not be empty"); + } + if !value + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '-')) + { + anyhow::bail!("{kind} '{value}' may contain only ASCII letters, digits, '.', '_' and '-'"); + } + Ok(()) +} + +fn slug(value: &str) -> String { + let mut output = String::new(); + let mut previous_separator = false; + for character in value.chars() { + let normalized = character.to_ascii_lowercase(); + if normalized.is_ascii_alphanumeric() || matches!(normalized, '_' | '-') { + output.push(normalized); + previous_separator = false; + } else if !previous_separator { + output.push('-'); + previous_separator = true; + } + } + output.trim_matches('-').to_string() +} + +/// Expected reproducibility behavior for a provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolDeterminism { + Deterministic, + ConfigurationDependent, + Nondeterministic, +} + +/// Whether a provider runs locally or depends on network/service behavior. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolLocality { + Local, + LocalService, + NetworkOptional, + NetworkRequired, +} + +/// Expected fidelity/loss behavior for the provider as a whole. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolFidelity { + Lossless, + PartialLoss, + Lossy, + PathDependent, +} + +/// Fleet/support importance shown by `doctor`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum ToolSupportTier { + Required, + #[default] + Optional, + Experimental, +} + +impl ToolSupportTier { + pub fn as_str(self) -> &'static str { + match self { + Self::Required => "required", + Self::Optional => "optional", + Self::Experimental => "experimental", + } + } +} + +/// How a tool/provider is discovered at runtime. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ToolDiscovery { + Executable { + candidates: Vec, + #[serde(default = "default_version_args")] + version_args: Vec, + }, + RuntimeService { + service: String, + }, + Virtual, +} + +fn default_version_args() -> Vec { + vec!["--version".to_string()] +} + +/// Minimum/maximum accepted semantic-ish numeric version. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolVersionRequirement { + #[serde(default)] + pub min_inclusive: Option, + #[serde(default)] + pub max_exclusive: Option, +} + +impl ToolVersionRequirement { + pub fn is_unconstrained(&self) -> bool { + self.min_inclusive.is_none() && self.max_exclusive.is_none() + } + + fn supports(&self, version_line: Option<&str>) -> Result { + if self.is_unconstrained() { + return Ok(true); + } + let installed = version_line + .and_then(parse_numeric_version) + .ok_or_else(|| anyhow::anyhow!("version probe did not contain a numeric version"))?; + + if let Some(minimum) = &self.min_inclusive { + let minimum = parse_numeric_version(minimum) + .ok_or_else(|| anyhow::anyhow!("invalid registry minimum version '{minimum}'"))?; + if installed < minimum { + return Ok(false); + } + } + if let Some(maximum) = &self.max_exclusive { + let maximum = parse_numeric_version(maximum) + .ok_or_else(|| anyhow::anyhow!("invalid registry maximum version '{maximum}'"))?; + if installed >= maximum { + return Ok(false); + } + } + Ok(true) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct NumericVersion([u64; 3]); + +fn parse_numeric_version(text: &str) -> Option { + for token in text.split(|character: char| !(character.is_ascii_digit() || character == '.')) { + let token = token.trim_matches('.'); + if token.is_empty() || !token.chars().next().is_some_and(|c| c.is_ascii_digit()) { + continue; + } + let mut parts = [0_u64; 3]; + let mut parsed_any = false; + for (index, part) in token.split('.').take(3).enumerate() { + if part.is_empty() { + break; + } + let Ok(value) = part.parse::() else { + break; + }; + parts[index] = value; + parsed_any = true; + } + if parsed_any { + return Some(NumericVersion(parts)); + } + } + None +} + +fn normalized_numeric_version(text: &str) -> Option { + parse_numeric_version(text) + .map(|version| format!("{}.{}.{}", version.0[0], version.0[1], version.0[2])) +} + +/// Environment requirement without storing or exposing the value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolEnvironmentRequirement { + pub name: String, + #[serde(default)] + pub credential: bool, +} + +/// Canonical provider/tool descriptor. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolDescriptor { + pub id: ToolId, + pub name: String, + pub discovery: ToolDiscovery, + #[serde(default)] + pub version: ToolVersionRequirement, + #[serde(default)] + pub operating_systems: Vec, + #[serde(default)] + pub architectures: Vec, + #[serde(default)] + pub capabilities: Vec, + #[serde(default)] + pub input_media_types: Vec, + #[serde(default)] + pub output_media_types: Vec, + pub determinism: ToolDeterminism, + pub locality: ToolLocality, + pub fidelity: ToolFidelity, + #[serde(default)] + pub required_environment: Vec, + #[serde(default)] + pub required_configuration: Vec, + #[serde(default)] + pub required_services: Vec, + #[serde(default)] + pub fallbacks: Vec, + #[serde(default)] + pub support_tier: ToolSupportTier, + #[serde(default)] + pub license_notes: Option, + #[serde(default)] + pub distribution_notes: Option, +} + +impl ToolDescriptor { + pub fn command(id: ToolId, executable: impl Into) -> Self { + let executable = executable.into(); + Self { + name: executable.clone(), + id, + discovery: ToolDiscovery::Executable { + candidates: vec![executable], + version_args: default_version_args(), + }, + version: ToolVersionRequirement::default(), + operating_systems: Vec::new(), + architectures: Vec::new(), + capabilities: Vec::new(), + input_media_types: Vec::new(), + output_media_types: Vec::new(), + determinism: ToolDeterminism::ConfigurationDependent, + locality: ToolLocality::Local, + fidelity: ToolFidelity::PathDependent, + required_environment: Vec::new(), + required_configuration: Vec::new(), + required_services: Vec::new(), + fallbacks: Vec::new(), + support_tier: ToolSupportTier::Experimental, + license_notes: None, + distribution_notes: Some( + "Dynamically registered command provider; distribution is owned by the host environment" + .to_string(), + ), + } + } + + pub fn virtual_provider(id: ToolId, name: impl Into) -> Self { + Self { + name: name.into(), + id, + discovery: ToolDiscovery::Virtual, + version: ToolVersionRequirement::default(), + operating_systems: Vec::new(), + architectures: Vec::new(), + capabilities: Vec::new(), + input_media_types: Vec::new(), + output_media_types: Vec::new(), + determinism: ToolDeterminism::ConfigurationDependent, + locality: ToolLocality::NetworkOptional, + fidelity: ToolFidelity::PathDependent, + required_environment: Vec::new(), + required_configuration: Vec::new(), + required_services: Vec::new(), + fallbacks: Vec::new(), + support_tier: ToolSupportTier::Experimental, + license_notes: None, + distribution_notes: None, + } + } +} + +#[derive(Debug, Deserialize)] +struct ToolRegistryDocument { + schema: String, + tools: Vec, +} + +/// Runtime facts used to evaluate non-executable requirements. +#[derive(Debug, Clone)] +pub struct ToolRuntimeContext { + pub operating_system: String, + pub architecture: String, + environment_names: BTreeSet, + configuration_keys: BTreeSet, + runtime_services: BTreeSet, +} + +impl ToolRuntimeContext { + pub fn current() -> Self { + Self { + operating_system: env::consts::OS.to_string(), + architecture: env::consts::ARCH.to_string(), + environment_names: env::vars_os() + .map(|(name, _)| name.to_string_lossy().to_ascii_uppercase()) + .collect(), + configuration_keys: BTreeSet::new(), + runtime_services: BTreeSet::new(), + } + } + + pub fn for_platform(os: impl Into, architecture: impl Into) -> Self { + Self { + operating_system: os.into(), + architecture: architecture.into(), + environment_names: BTreeSet::new(), + configuration_keys: BTreeSet::new(), + runtime_services: BTreeSet::new(), + } + } + + pub fn with_environment(mut self, name: impl Into) -> Self { + self.environment_names + .insert(name.into().to_ascii_uppercase()); + self + } + + pub fn with_configuration(mut self, key: impl Into) -> Self { + self.configuration_keys.insert(key.into()); + self + } + + pub fn with_runtime_service(mut self, service: impl Into) -> Self { + self.runtime_services.insert(service.into()); + self + } +} + +/// Probe seam used by tests and embedders to avoid arbitrary host-state dependencies. +pub trait ToolProbe: Send + Sync { + fn probe(&self, executable: &str, version_args: &[String]) -> ToolProbeEvidence; +} + +#[derive(Clone, Default)] +pub struct ProcessToolProbe { + executor: ProcessExecutor, +} + +impl ProcessToolProbe { + pub fn new() -> Self { + Self::default() + } +} + +impl ToolProbe for ProcessToolProbe { + fn probe(&self, executable: &str, version_args: &[String]) -> ToolProbeEvidence { + self.executor + .probe_version_with_args(executable, version_args) + } +} + +/// Canonical availability state used by planner, doctor, SDK, and CLI output. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolAvailabilityStatus { + Available, + MissingExecutable, + UnsupportedVersion, + MissingRuntimeService, + MissingCredential, + MissingConfiguration, + UnsupportedPlatform, + ProbeFailed, + UnknownProvider, +} + +impl ToolAvailabilityStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::MissingExecutable => "missing_executable", + Self::UnsupportedVersion => "unsupported_version", + Self::MissingRuntimeService => "missing_runtime_service", + Self::MissingCredential => "missing_credential", + Self::MissingConfiguration => "missing_configuration", + Self::UnsupportedPlatform => "unsupported_platform", + Self::ProbeFailed => "probe_failed", + Self::UnknownProvider => "unknown_provider", + } + } +} + +/// One evaluated tool/provider. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolAvailability { + pub id: ToolId, + pub name: String, + pub status: ToolAvailabilityStatus, + pub support_tier: ToolSupportTier, + pub selected_executable: Option, + pub version_line: Option, + pub normalized_version: Option, + pub diagnostic: Option, +} + +impl ToolAvailability { + pub fn is_available(&self) -> bool { + self.status == ToolAvailabilityStatus::Available + } + + pub fn summary(&self) -> String { + let detail = self + .diagnostic + .as_deref() + .or(self.version_line.as_deref()) + .unwrap_or("no additional detail"); + format!("{}: {} ({detail})", self.id, self.status.as_str()) + } +} + +/// Deterministically sorted set of evaluated tools. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolInventory { + pub tools: Vec, +} + +impl ToolInventory { + pub fn get(&self, id: &str) -> Option<&ToolAvailability> { + self.tools.iter().find(|tool| tool.id.as_str() == id) + } + + pub fn available_ids(&self) -> HashSet { + self.tools + .iter() + .filter(|tool| tool.is_available()) + .map(|tool| tool.id.to_string()) + .collect() + } + + pub fn blocked_summaries(&self) -> Vec { + self.tools + .iter() + .filter(|tool| !tool.is_available()) + .map(ToolAvailability::summary) + .collect() + } +} + +/// Evidence for a selected provider included in a toolchain fingerprint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SelectedToolEvidence { + pub id: ToolId, + pub executable: Option, + pub version: Option, + pub capabilities: Vec, + pub determinism: ToolDeterminism, + pub locality: ToolLocality, +} + +/// Reproducibility evidence derived only from providers selected by a plan/run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolchainSnapshot { + pub schema: String, + pub fingerprint: String, + pub operating_system: String, + pub architecture: String, + pub selected_tools: Vec, +} + +#[derive(Serialize)] +struct FingerprintMaterial<'a> { + schema: &'a str, + operating_system: &'a str, + architecture: &'a str, + selected_tools: &'a [SelectedToolEvidence], +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolResolutionDecision { + pub id: ToolId, + pub status: ToolAvailabilityStatus, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolResolution { + pub requested: ToolId, + pub selected: Option, + pub decisions: Vec, +} + +/// Public registry that can be extended by plugins without exposing planner internals. +#[derive(Debug, Clone)] +pub struct ToolRegistry { + schema: String, + tools: BTreeMap, +} + +impl ToolRegistry { + pub fn new() -> Self { + Self { + schema: TOOL_REGISTRY_SCHEMA.to_string(), + tools: BTreeMap::new(), + } + } + + pub fn builtins() -> Self { + Self::from_yaml(include_str!("../data/tool-registry.yaml")) + .expect("embedded Renderflow tool registry must be valid") + } + + pub fn from_yaml(yaml: &str) -> Result { + let document: ToolRegistryDocument = + serde_yaml_ng::from_str(yaml).context("failed to parse tool registry YAML")?; + if document.schema != TOOL_REGISTRY_SCHEMA { + anyhow::bail!( + "unsupported tool registry schema '{}'; expected '{}'", + document.schema, + TOOL_REGISTRY_SCHEMA + ); + } + let mut registry = Self { + schema: document.schema, + tools: BTreeMap::new(), + }; + for descriptor in document.tools { + registry.register(descriptor)?; + } + Ok(registry) + } + + pub fn schema(&self) -> &str { + &self.schema + } + + pub fn register(&mut self, mut descriptor: ToolDescriptor) -> Result<&mut Self> { + validate_identifier("tool id", descriptor.id.as_str())?; + for capability in &descriptor.capabilities { + validate_identifier("capability id", capability.as_str())?; + } + descriptor.capabilities.sort(); + descriptor.capabilities.dedup(); + descriptor.fallbacks.sort(); + descriptor.fallbacks.dedup(); + self.tools.insert(descriptor.id.clone(), descriptor); + Ok(self) + } + + pub fn contains(&self, id: &ToolId) -> bool { + self.tools.contains_key(id) + } + + pub fn get(&self, id: &str) -> Option<&ToolDescriptor> { + self.tools.values().find(|tool| tool.id.as_str() == id) + } + + pub fn all(&self) -> impl Iterator { + self.tools.values() + } + + pub fn capabilities(&self) -> BTreeMap> { + let mut capabilities: BTreeMap> = BTreeMap::new(); + for descriptor in self.tools.values() { + for capability in &descriptor.capabilities { + capabilities + .entry(capability.to_string()) + .or_default() + .push(descriptor.id.to_string()); + } + } + for providers in capabilities.values_mut() { + providers.sort(); + providers.dedup(); + } + capabilities + } + + pub fn canonical_id_for_executable(&self, executable: &str) -> ToolId { + for descriptor in self.tools.values() { + if let ToolDiscovery::Executable { candidates, .. } = &descriptor.discovery { + if candidates.iter().any(|candidate| candidate == executable) { + return descriptor.id.clone(); + } + } + } + canonical_tool_id_for_hint(executable) + } + + pub fn ensure_command_provider( + &mut self, + id: ToolId, + executable: impl Into, + capability: CapabilityId, + ) -> Result<&mut Self> { + if !self.tools.contains_key(&id) { + self.register(ToolDescriptor::command(id.clone(), executable))?; + } + self.add_capability(&id, capability)?; + Ok(self) + } + + pub fn ensure_virtual_provider( + &mut self, + id: ToolId, + name: impl Into, + capability: CapabilityId, + ) -> Result<&mut Self> { + if !self.tools.contains_key(&id) { + self.register(ToolDescriptor::virtual_provider(id.clone(), name))?; + } + self.add_capability(&id, capability)?; + Ok(self) + } + + pub fn add_capability(&mut self, id: &ToolId, capability: CapabilityId) -> Result<&mut Self> { + let descriptor = self + .tools + .get_mut(id) + .ok_or_else(|| anyhow::anyhow!("tool provider '{}' is not registered", id))?; + if !descriptor.capabilities.contains(&capability) { + descriptor.capabilities.push(capability); + descriptor.capabilities.sort(); + } + Ok(self) + } + + pub fn assess_all_with( + &self, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, + ) -> ToolInventory { + let ids: Vec = self.tools.keys().map(ToString::to_string).collect(); + self.assess_ids_with(ids, probe, context) + } + + pub fn assess_all_current(&self) -> ToolInventory { + self.assess_all_with(&ProcessToolProbe::new(), &ToolRuntimeContext::current()) + } + + pub fn assess_ids_with( + &self, + ids: I, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, + ) -> ToolInventory + where + I: IntoIterator, + S: AsRef, + { + let unique: BTreeSet = ids.into_iter().map(|id| id.as_ref().to_string()).collect(); + let mut tools = Vec::with_capacity(unique.len()); + for id in unique { + if let Some(descriptor) = self.get(&id) { + tools.push(evaluate_descriptor(descriptor, probe, context)); + } else { + let tool_id = ToolId::new(id.clone()) + .unwrap_or_else(|_| ToolId(format!("tool.unknown.{}", slug(&id)))); + tools.push(ToolAvailability { + id: tool_id, + name: id.clone(), + status: ToolAvailabilityStatus::UnknownProvider, + support_tier: ToolSupportTier::Experimental, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: Some(format!("provider '{id}' is not registered")), + }); + } + } + ToolInventory { tools } + } + + pub fn assess_ids_current(&self, ids: I) -> ToolInventory + where + I: IntoIterator, + S: AsRef, + { + self.assess_ids_with( + ids, + &ProcessToolProbe::new(), + &ToolRuntimeContext::current(), + ) + } + + pub fn fingerprint_selected( + &self, + inventory: &ToolInventory, + ids: I, + context: &ToolRuntimeContext, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + let ids: BTreeSet = ids.into_iter().map(|id| id.as_ref().to_string()).collect(); + let mut selected_tools = Vec::with_capacity(ids.len()); + for id in ids { + let availability = inventory + .get(&id) + .ok_or_else(|| anyhow::anyhow!("tool '{}' was not assessed", id))?; + if !availability.is_available() { + anyhow::bail!( + "cannot fingerprint unavailable provider '{}': {}", + id, + availability.summary() + ); + } + let descriptor = self + .get(&id) + .ok_or_else(|| anyhow::anyhow!("tool '{}' is not registered", id))?; + selected_tools.push(SelectedToolEvidence { + id: descriptor.id.clone(), + executable: availability.selected_executable.clone(), + version: availability + .normalized_version + .clone() + .or_else(|| availability.version_line.clone()), + capabilities: descriptor.capabilities.clone(), + determinism: descriptor.determinism, + locality: descriptor.locality, + }); + } + selected_tools.sort_by(|left, right| left.id.cmp(&right.id)); + + let material = FingerprintMaterial { + schema: TOOLCHAIN_SCHEMA, + operating_system: &context.operating_system, + architecture: &context.architecture, + selected_tools: &selected_tools, + }; + let encoded = serde_json::to_vec(&material) + .context("failed to serialize toolchain fingerprint material")?; + let digest = Sha256::digest(encoded); + Ok(ToolchainSnapshot { + schema: TOOLCHAIN_SCHEMA.to_string(), + fingerprint: format!("sha256:{digest:x}"), + operating_system: context.operating_system.clone(), + architecture: context.architecture.clone(), + selected_tools, + }) + } + + pub fn fingerprint_for_dag( + &self, + inventory: &ToolInventory, + dag: &MultiTargetDag, + context: &ToolRuntimeContext, + ) -> Result { + self.fingerprint_selected( + inventory, + dag.all_edges() + .iter() + .filter_map(|edge| edge.provider_id.as_deref()), + context, + ) + } + + pub fn resolve_with( + &self, + requested: &ToolId, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, + ) -> ToolResolution { + let mut queue = vec![requested.clone()]; + let mut visited = BTreeSet::new(); + let mut decisions = Vec::new(); + + while let Some(id) = queue.first().cloned() { + queue.remove(0); + if !visited.insert(id.clone()) { + continue; + } + let Some(descriptor) = self.tools.get(&id) else { + decisions.push(ToolResolutionDecision { + id, + status: ToolAvailabilityStatus::UnknownProvider, + reason: "provider is not registered".to_string(), + }); + continue; + }; + let availability = evaluate_descriptor(descriptor, probe, context); + decisions.push(ToolResolutionDecision { + id: id.clone(), + status: availability.status, + reason: availability.summary(), + }); + if availability.is_available() { + return ToolResolution { + requested: requested.clone(), + selected: Some(id), + decisions, + }; + } + queue.extend(descriptor.fallbacks.iter().cloned()); + } + + ToolResolution { + requested: requested.clone(), + selected: None, + decisions, + } + } +} + +impl Default for ToolRegistry { + fn default() -> Self { + Self::new() + } +} + +fn evaluate_descriptor( + descriptor: &ToolDescriptor, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, +) -> ToolAvailability { + let unavailable = |status: ToolAvailabilityStatus, diagnostic: String| ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status, + support_tier: descriptor.support_tier, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: Some(diagnostic), + }; + + if !descriptor.operating_systems.is_empty() + && !descriptor + .operating_systems + .iter() + .any(|os| os == &context.operating_system) + { + return unavailable( + ToolAvailabilityStatus::UnsupportedPlatform, + format!( + "platform '{}' is unsupported; expected one of {}", + context.operating_system, + descriptor.operating_systems.join(", ") + ), + ); + } + if !descriptor.architectures.is_empty() + && !descriptor + .architectures + .iter() + .any(|architecture| architecture == &context.architecture) + { + return unavailable( + ToolAvailabilityStatus::UnsupportedPlatform, + format!( + "architecture '{}' is unsupported; expected one of {}", + context.architecture, + descriptor.architectures.join(", ") + ), + ); + } + + for requirement in &descriptor.required_environment { + if !context + .environment_names + .contains(&requirement.name.to_ascii_uppercase()) + { + return unavailable( + if requirement.credential { + ToolAvailabilityStatus::MissingCredential + } else { + ToolAvailabilityStatus::MissingConfiguration + }, + format!( + "required environment variable '{}' is not set", + requirement.name + ), + ); + } + } + for key in &descriptor.required_configuration { + if !context.configuration_keys.contains(key) { + return unavailable( + ToolAvailabilityStatus::MissingConfiguration, + format!("required configuration key '{key}' is not available"), + ); + } + } + for service in &descriptor.required_services { + if !context.runtime_services.contains(service) { + return unavailable( + ToolAvailabilityStatus::MissingRuntimeService, + format!("required runtime service '{service}' is unavailable"), + ); + } + } + + match &descriptor.discovery { + ToolDiscovery::Virtual => ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::Available, + support_tier: descriptor.support_tier, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: None, + }, + ToolDiscovery::RuntimeService { service } => { + if context.runtime_services.contains(service) { + ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::Available, + support_tier: descriptor.support_tier, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: Some(format!("runtime service '{service}' is available")), + } + } else { + unavailable( + ToolAvailabilityStatus::MissingRuntimeService, + format!("runtime service '{service}' is unavailable"), + ) + } + } + ToolDiscovery::Executable { + candidates, + version_args, + } => { + let mut saw_probe_failure = None; + for executable in candidates { + let evidence = probe.probe(executable, version_args); + match evidence.status { + ProcessProbeStatus::Available => { + let version_line = evidence.version_line.clone(); + match descriptor.version.supports(version_line.as_deref()) { + Ok(true) => { + return ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::Available, + support_tier: descriptor.support_tier, + selected_executable: Some(executable.clone()), + normalized_version: version_line + .as_deref() + .and_then(normalized_numeric_version), + version_line, + diagnostic: None, + }; + } + Ok(false) => { + return ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::UnsupportedVersion, + support_tier: descriptor.support_tier, + selected_executable: Some(executable.clone()), + normalized_version: version_line + .as_deref() + .and_then(normalized_numeric_version), + diagnostic: Some(format!( + "installed version does not satisfy registry requirement {:?}", + descriptor.version + )), + version_line, + }; + } + Err(error) => { + return unavailable( + ToolAvailabilityStatus::UnsupportedVersion, + error.to_string(), + ); + } + } + } + ProcessProbeStatus::Missing => {} + ProcessProbeStatus::TimedOut | ProcessProbeStatus::Failed => { + saw_probe_failure = Some( + evidence + .diagnostic + .unwrap_or_else(|| "version probe failed".to_string()), + ); + } + } + } + if let Some(diagnostic) = saw_probe_failure { + unavailable(ToolAvailabilityStatus::ProbeFailed, diagnostic) + } else { + unavailable( + ToolAvailabilityStatus::MissingExecutable, + format!("none of [{}] were found", candidates.join(", ")), + ) + } + } + } +} + +/// Map known executable/tool hints to stable built-in IDs; arbitrary hints get +/// a stable dynamic-command namespace. +pub fn canonical_tool_id_for_hint(hint: &str) -> ToolId { + let canonical = match hint { + "pandoc" => "tool.pandoc".to_string(), + "tectonic" => "tool.tectonic".to_string(), + "ffmpeg" => "tool.ffmpeg".to_string(), + "wkhtmltopdf" => "tool.wkhtmltopdf".to_string(), + "zip" => "tool.zip".to_string(), + "img2pdf" => "tool.img2pdf".to_string(), + "gs" | "ghostscript" => "tool.ghostscript".to_string(), + value => format!("tool.command.{}", slug(value)), + }; + ToolId::new(canonical).expect("canonicalized tool id is valid") +} + +/// Stable capability ID for a concrete format transformation. +pub fn transform_capability_id(from: Format, to: Format) -> CapabilityId { + CapabilityId::new(format!("transform.{}.{}", from, to)) + .expect("Format display values produce valid capability ids") +} + +/// Assess all provider IDs referenced by a graph and return an availability-filtered graph. +pub fn filter_graph_for_current_toolchain( + graph: &TransformGraph, + registry: &ToolRegistry, +) -> (TransformGraph, ToolInventory, ToolRuntimeContext) { + let context = ToolRuntimeContext::current(); + let inventory = + registry.assess_ids_with(graph.provider_ids(), &ProcessToolProbe::new(), &context); + let available = inventory.available_ids(); + let filtered = graph.filtered_by_available_providers(&available); + (filtered, inventory, context) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::process::{ProcessPlatform, ProcessTreeTermination}; + + #[derive(Default)] + struct FakeProbe { + responses: BTreeMap, + } + + impl FakeProbe { + fn with( + mut self, + executable: &str, + status: ProcessProbeStatus, + version: Option<&str>, + ) -> Self { + self.responses.insert( + executable.to_string(), + ToolProbeEvidence { + executable: executable.to_string(), + status, + version_line: version.map(str::to_string), + duration_ms: 1, + platform: ProcessPlatform { + os: "test", + arch: "test", + tree_termination: ProcessTreeTermination::DirectChild, + }, + diagnostic: None, + }, + ); + self + } + } + + impl ToolProbe for FakeProbe { + fn probe(&self, executable: &str, _version_args: &[String]) -> ToolProbeEvidence { + self.responses + .get(executable) + .cloned() + .unwrap_or(ToolProbeEvidence { + executable: executable.to_string(), + status: ProcessProbeStatus::Missing, + version_line: None, + duration_ms: 0, + platform: ProcessPlatform { + os: "test", + arch: "test", + tree_termination: ProcessTreeTermination::DirectChild, + }, + diagnostic: Some("missing".to_string()), + }) + } + } + + fn test_descriptor(id: &str, executable: &str) -> ToolDescriptor { + ToolDescriptor { + id: ToolId::new(id).unwrap(), + name: id.to_string(), + discovery: ToolDiscovery::Executable { + candidates: vec![executable.to_string()], + version_args: vec!["--version".to_string()], + }, + version: ToolVersionRequirement::default(), + operating_systems: Vec::new(), + architectures: Vec::new(), + capabilities: vec![CapabilityId::new("transform.test").unwrap()], + input_media_types: Vec::new(), + output_media_types: Vec::new(), + determinism: ToolDeterminism::Deterministic, + locality: ToolLocality::Local, + fidelity: ToolFidelity::Lossless, + required_environment: Vec::new(), + required_configuration: Vec::new(), + required_services: Vec::new(), + fallbacks: Vec::new(), + support_tier: ToolSupportTier::Optional, + license_notes: None, + distribution_notes: None, + } + } + + #[test] + fn embedded_registry_contains_core_wrapped_tools() { + let registry = ToolRegistry::builtins(); + assert!(registry.get("tool.pandoc").is_some()); + assert!(registry.get("tool.tectonic").is_some()); + assert!(registry.get("tool.ffmpeg").is_some()); + assert!(registry.get("tool.ghostscript").is_some()); + } + + #[test] + fn stable_ids_reject_whitespace() { + assert!(ToolId::new("tool.bad id").is_err()); + assert!(CapabilityId::new("bad capability").is_err()); + } + + #[test] + fn missing_executable_is_distinct() { + let mut registry = ToolRegistry::new(); + registry + .register(test_descriptor("tool.fake", "fake")) + .unwrap(); + let inventory = registry.assess_all_with( + &FakeProbe::default(), + &ToolRuntimeContext::for_platform("linux", "x86_64"), + ); + assert_eq!( + inventory.tools[0].status, + ToolAvailabilityStatus::MissingExecutable + ); + } + + #[test] + fn unsupported_version_is_distinct() { + let mut descriptor = test_descriptor("tool.fake", "fake"); + descriptor.version.min_inclusive = Some("2.0.0".to_string()); + let mut registry = ToolRegistry::new(); + registry.register(descriptor).unwrap(); + let probe = + FakeProbe::default().with("fake", ProcessProbeStatus::Available, Some("fake 1.9.0")); + let inventory = + registry.assess_all_with(&probe, &ToolRuntimeContext::for_platform("linux", "x86_64")); + assert_eq!( + inventory.tools[0].status, + ToolAvailabilityStatus::UnsupportedVersion + ); + } + + #[test] + fn doctor_states_distinguish_service_credential_config_and_platform() { + let mut registry = ToolRegistry::new(); + + let mut service = + ToolDescriptor::virtual_provider(ToolId::new("tool.service").unwrap(), "service"); + service.discovery = ToolDiscovery::RuntimeService { + service: "daemon".to_string(), + }; + registry.register(service).unwrap(); + + let mut credential = + ToolDescriptor::virtual_provider(ToolId::new("tool.credential").unwrap(), "credential"); + credential + .required_environment + .push(ToolEnvironmentRequirement { + name: "FAKE_TOKEN".to_string(), + credential: true, + }); + registry.register(credential).unwrap(); + + let mut configuration = ToolDescriptor::virtual_provider( + ToolId::new("tool.configuration").unwrap(), + "configuration", + ); + configuration + .required_configuration + .push("profile".to_string()); + registry.register(configuration).unwrap(); + + let mut platform = + ToolDescriptor::virtual_provider(ToolId::new("tool.platform").unwrap(), "platform"); + platform.operating_systems = vec!["plan9".to_string()]; + registry.register(platform).unwrap(); + + let inventory = registry.assess_all_with( + &FakeProbe::default(), + &ToolRuntimeContext::for_platform("linux", "x86_64"), + ); + assert_eq!( + inventory.get("tool.service").unwrap().status, + ToolAvailabilityStatus::MissingRuntimeService + ); + assert_eq!( + inventory.get("tool.credential").unwrap().status, + ToolAvailabilityStatus::MissingCredential + ); + assert_eq!( + inventory.get("tool.configuration").unwrap().status, + ToolAvailabilityStatus::MissingConfiguration + ); + assert_eq!( + inventory.get("tool.platform").unwrap().status, + ToolAvailabilityStatus::UnsupportedPlatform + ); + } + + #[test] + fn fallback_resolution_explains_why_substitute_won() { + let mut primary = test_descriptor("tool.primary", "primary"); + primary + .fallbacks + .push(ToolId::new("tool.fallback").unwrap()); + let fallback = test_descriptor("tool.fallback", "fallback"); + let mut registry = ToolRegistry::new(); + registry.register(primary).unwrap(); + registry.register(fallback).unwrap(); + let probe = FakeProbe::default().with( + "fallback", + ProcessProbeStatus::Available, + Some("fallback 3.0.0"), + ); + let resolution = registry.resolve_with( + &ToolId::new("tool.primary").unwrap(), + &probe, + &ToolRuntimeContext::for_platform("linux", "x86_64"), + ); + assert_eq!( + resolution.selected.as_ref().map(ToolId::as_str), + Some("tool.fallback") + ); + assert_eq!(resolution.decisions.len(), 2); + assert_eq!( + resolution.decisions[0].status, + ToolAvailabilityStatus::MissingExecutable + ); + } + + #[test] + fn fingerprint_is_order_independent_and_selected_only() { + let mut registry = ToolRegistry::new(); + registry.register(test_descriptor("tool.a", "a")).unwrap(); + registry.register(test_descriptor("tool.b", "b")).unwrap(); + let probe = FakeProbe::default() + .with("a", ProcessProbeStatus::Available, Some("a 1.2.3")) + .with("b", ProcessProbeStatus::Available, Some("b 4.5.6")); + let context = ToolRuntimeContext::for_platform("linux", "x86_64"); + let inventory = registry.assess_all_with(&probe, &context); + let first = registry + .fingerprint_selected(&inventory, ["tool.b", "tool.a"], &context) + .unwrap(); + let second = registry + .fingerprint_selected(&inventory, ["tool.a", "tool.b"], &context) + .unwrap(); + let only_a = registry + .fingerprint_selected(&inventory, ["tool.a"], &context) + .unwrap(); + assert_eq!(first.fingerprint, second.fingerprint); + assert_ne!(first.fingerprint, only_a.fingerprint); + } + + #[test] + fn dynamic_command_provider_can_be_augmented_with_transform_capability() { + let mut registry = ToolRegistry::builtins(); + let id = registry.canonical_id_for_executable("made-up-renderer"); + registry + .ensure_command_provider( + id.clone(), + "made-up-renderer", + CapabilityId::new("transform.markdown.pdf").unwrap(), + ) + .unwrap(); + let descriptor = registry.get(id.as_str()).unwrap(); + assert!(descriptor + .capabilities + .iter() + .any(|capability| capability.as_str() == "transform.markdown.pdf")); + } +} diff --git a/crates/renderflow-core/src/transforms/aggregation.rs b/crates/renderflow-core/src/transforms/aggregation.rs index cc6096e..4651681 100644 --- a/crates/renderflow-core/src/transforms/aggregation.rs +++ b/crates/renderflow-core/src/transforms/aggregation.rs @@ -8,7 +8,7 @@ use anyhow::{Context, Result}; use tracing::{debug, info}; use crate::process::{ - is_explicit_shell_invocation, ProcessExpectedOutput, ProcessExecutor, ProcessInput, + is_explicit_shell_invocation, ProcessExecutor, ProcessExpectedOutput, ProcessInput, ProcessOutputMode, ProcessRequest, DEFAULT_CAPTURE_LIMIT_BYTES, DEFAULT_PROCESS_TIMEOUT, }; @@ -42,7 +42,9 @@ impl AggregationRegistry { } pub fn get(&self, name: &str) -> Option<&dyn AggregationTransform> { - self.transforms.get(name).map(|transform| transform.as_ref()) + self.transforms + .get(name) + .map(|transform| transform.as_ref()) } pub fn apply(&self, name: &str, inputs: &[&str], output_path: &str) -> Result<()> { @@ -266,7 +268,10 @@ mod tests { registry .apply("join", &["page1", "page2", "page3"], out.to_str().unwrap()) .unwrap(); - assert_eq!(std::fs::read_to_string(&out).unwrap(), "page1\npage2\npage3"); + assert_eq!( + std::fs::read_to_string(&out).unwrap(), + "page1\npage2\npage3" + ); } #[test] @@ -297,9 +302,7 @@ mod tests { ); let dir = tempfile::tempdir().unwrap(); let out = dir.path().join("out.txt"); - assert!(transform - .aggregate(&["a"], out.to_str().unwrap()) - .is_err()); + assert!(transform.aggregate(&["a"], out.to_str().unwrap()).is_err()); } #[cfg(unix)] diff --git a/crates/renderflow-core/src/transforms/command.rs b/crates/renderflow-core/src/transforms/command.rs index 2c68f51..d6ab6f5 100644 --- a/crates/renderflow-core/src/transforms/command.rs +++ b/crates/renderflow-core/src/transforms/command.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use super::Transform; use crate::process::{ - is_explicit_shell_invocation, ProcessExpectedOutput, ProcessExecutor, ProcessInput, + is_explicit_shell_invocation, ProcessExecutor, ProcessExpectedOutput, ProcessInput, ProcessOutputMode, ProcessRequest, DEFAULT_CAPTURE_LIMIT_BYTES, DEFAULT_PROCESS_TIMEOUT, }; diff --git a/crates/renderflow-core/src/transforms/yaml_loader.rs b/crates/renderflow-core/src/transforms/yaml_loader.rs index 1d02def..7938efd 100644 --- a/crates/renderflow-core/src/transforms/yaml_loader.rs +++ b/crates/renderflow-core/src/transforms/yaml_loader.rs @@ -12,6 +12,7 @@ use super::{ Transform, TransformRegistry, }; use crate::ai::providers::{OllamaProvider, OpenAiProvider}; +use crate::toolchain::{transform_capability_id, ToolId, ToolRegistry}; /// Top-level structure of a YAML transform configuration file. /// @@ -62,6 +63,9 @@ pub struct YamlTransformDef { /// those fields is set. #[serde(default)] pub program: Option, + /// Optional stable provider ID. When omitted, Renderflow infers one from program/plugin/AI. + #[serde(default)] + pub provider: Option, /// Arguments passed to the program. /// /// Use `{input}` as a placeholder for a temporary file that contains the @@ -176,6 +180,11 @@ impl YamlTransformDef { if self.name.trim().is_empty() { anyhow::bail!("transform 'name' must not be empty"); } + if let Some(provider) = &self.provider { + ToolId::new(provider.clone()).with_context(|| { + format!("transform '{}': invalid stable provider id", self.name) + })?; + } match (&self.ai, &self.plugin, &self.program) { // ai takes precedence – validate the backend name. (Some(backend), _, _) if backend.trim().is_empty() => { @@ -364,6 +373,36 @@ impl YamlTransformDef { .unwrap_or(false) } + /// Resolve the stable provider ID used by planning/toolchain evidence. + pub fn provider_id(&self, registry: &ToolRegistry) -> Result { + if let Some(provider) = &self.provider { + return ToolId::new(provider.clone()); + } + if let Some(program) = &self.program { + return Ok(registry.canonical_id_for_executable(program)); + } + if let Some(plugin) = &self.plugin { + let component = plugin + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { + character.to_ascii_lowercase() + } else { + '-' + } + }) + .collect::(); + return ToolId::new(format!("tool.plugin.{component}")); + } + if let Some(ai) = &self.ai { + return ToolId::new(format!("tool.ai.{}", ai.to_ascii_lowercase())); + } + anyhow::bail!( + "transform '{}': cannot resolve provider identity", + self.name + ) + } + /// Build a [`CommandAggregationTransform`] from this definition. /// /// Only valid when `program` is set; call [`validate`](Self::validate) @@ -540,34 +579,75 @@ pub fn load_aggregation_transforms_from_yaml(path: &str) -> Result Result { + let content = fs::read_to_string(path) + .with_context(|| format!("Failed to read transform config: {}", path))?; + parse_tool_registry_from_str(&content) + .with_context(|| format!("Failed to load tool registry from: {}", path)) +} + +/// Build the runtime provider registry implied by a transform YAML document. +pub fn parse_tool_registry_from_str(yaml: &str) -> Result { + let config: YamlTransformConfig = + serde_yaml_ng::from_str(yaml).context("Failed to parse YAML transform config")?; + let mut registry = ToolRegistry::builtins(); + + for def in &config.transforms { + def.validate()?; + let from: crate::graph::Format = def.from.parse()?; + let to: crate::graph::Format = def.to.parse()?; + let capability = transform_capability_id(from, to); + let provider = def.provider_id(®istry)?; + + if let Some(program) = &def.program { + registry.ensure_command_provider(provider, program.clone(), capability)?; + } else { + let provider_name = def + .plugin + .as_deref() + .or(def.ai.as_deref()) + .unwrap_or(def.name.as_str()); + registry.ensure_virtual_provider(provider, provider_name, capability)?; + } + } + + Ok(registry) +} + +/// Build graph, executor, and canonical runtime tool registry from a YAML file. +pub fn build_graph_executor_and_tools_from_yaml( path: &str, -) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { +) -> Result<( + crate::graph::TransformGraph, + crate::graph::DagExecutor, + ToolRegistry, +)> { let content = fs::read_to_string(path) .with_context(|| format!("Failed to read transform config: {}", path))?; - build_graph_and_executor_from_str(&content) - .with_context(|| format!("Failed to build graph and executor from: {}", path)) + build_graph_executor_and_tools_from_str(&content) + .with_context(|| format!("Failed to build graph/executor/tools from: {}", path)) } -/// Build a [`TransformGraph`] and a [`DagExecutor`] from a YAML string. -/// -/// See [`build_graph_and_executor_from_yaml`] for details. -pub fn build_graph_and_executor_from_str( +/// Build graph, executor, and canonical runtime tool registry from YAML text. +pub fn build_graph_executor_and_tools_from_str( yaml: &str, -) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { +) -> Result<( + crate::graph::TransformGraph, + crate::graph::DagExecutor, + ToolRegistry, +)> { use std::sync::Arc; use crate::graph::{DagExecutor, Format, InputKind, TransformEdge, TransformGraph}; let config: YamlTransformConfig = serde_yaml_ng::from_str(yaml).context("Failed to parse YAML transform config")?; - + let tool_registry = parse_tool_registry_from_str(yaml)?; let mut graph = TransformGraph::new(); let mut executor = DagExecutor::new(); for def in &config.transforms { def.validate()?; - let from: Format = def .from .parse() @@ -576,24 +656,27 @@ pub fn build_graph_and_executor_from_str( .to .parse() .with_context(|| format!("transform '{}': invalid 'to' format", def.name))?; - let input_kind = if def.is_collection() { InputKind::Collection } else { InputKind::Single }; + let provider = def.provider_id(&tool_registry)?; + let capability = transform_capability_id(from, to); - graph.add_transform(TransformEdge::with_input_kind( - from, - to, - def.cost, - def.quality, - input_kind, - )); + graph.add_transform( + TransformEdge::with_input_kind(from, to, def.cost, def.quality, input_kind) + .with_provider(provider.to_string(), capability.to_string()), + ); if def.is_collection() { let agg = Arc::new(def.to_aggregation_transform()?); executor.register_aggregation(from, to, agg); + } else if def.plugin.is_some() { + anyhow::bail!( + "transform '{}': graph execution requires plugin registration through an embedding registry", + def.name + ); } else { let transform: Arc = if def.ai.is_some() { Arc::new(def.to_ai_transform()?) @@ -604,6 +687,22 @@ pub fn build_graph_and_executor_from_str( } } + Ok((graph, executor, tool_registry)) +} + +/// Compatibility wrapper returning only graph and executor. +pub fn build_graph_and_executor_from_yaml( + path: &str, +) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { + let (graph, executor, _tools) = build_graph_executor_and_tools_from_yaml(path)?; + Ok((graph, executor)) +} + +/// Compatibility wrapper returning only graph and executor. +pub fn build_graph_and_executor_from_str( + yaml: &str, +) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { + let (graph, executor, _tools) = build_graph_executor_and_tools_from_str(yaml)?; Ok((graph, executor)) } diff --git a/docs/cli-reference/tools.md b/docs/cli-reference/tools.md new file mode 100644 index 0000000..15ecbbd --- /dev/null +++ b/docs/cli-reference/tools.md @@ -0,0 +1,37 @@ +# tools and capabilities + +Renderflow exposes the runtime provider registry through the same structured model used by planning and diagnostics. + +## List providers + +```bash +renderflow tools list +renderflow tools list --format json +renderflow tools list --format yaml +``` + +Use `--transforms ` to include providers inferred from a transform YAML file, including arbitrary command providers that are not part of the built-in catalog. + +## Inspect one provider + +```bash +renderflow tools inspect tool.ffmpeg +renderflow tools inspect tool.pandoc --format json +``` + +Inspection includes discovery strategy, live availability state, selected executable, installed version evidence, determinism/locality/fidelity metadata, capability IDs, fallbacks, and diagnostics. + +## List capabilities + +```bash +renderflow capabilities +renderflow capabilities --format json +``` + +Capability IDs and provider IDs are stable machine-readable identifiers. Human-readable CLI output is rendered from the same data returned by JSON/YAML modes. + +## Toolchain fingerprints + +Graph planning fingerprints only providers selected by the final DAG. The fingerprint includes the selected provider IDs, compatible installed versions, relevant executable identity, provider capability metadata, and the target OS/architecture. It does **not** hash the entire host environment. + +Graph execution uses that fingerprint in artifact-cache compatibility and writes the selected toolchain evidence into Renderflow state for reproducibility/provenance consumers. diff --git a/docs/user-guide/tool-registry.md b/docs/user-guide/tool-registry.md new file mode 100644 index 0000000..d349219 --- /dev/null +++ b/docs/user-guide/tool-registry.md @@ -0,0 +1,59 @@ +# Tool Registry + +!!! info + This page is generated from the canonical built-in catalog at + `crates/renderflow-core/data/tool-registry.yaml` by + `scripts/generate_tool_registry_doc.py`. Do not edit it by hand. + +Registry schema: `renderflow.tool-registry/v1` + +The built-in catalog defines stable provider IDs, discovery/version probes, +platform constraints, capability IDs, determinism/locality/fidelity metadata, +runtime requirements, fallback relationships, and distribution/licensing notes. +YAML command transforms and plugins can register additional runtime providers +without editing this built-in catalog. + +## Built-in providers + +| Provider ID | Name | Discovery | Tier | Determinism | Locality | +| --- | --- | --- | --- | --- | --- | +| `tool.ffmpeg` | FFmpeg | executable: `ffmpeg` | optional | configuration_dependent | local | +| `tool.ghostscript` | Ghostscript | executable: `gs` | experimental | configuration_dependent | local | +| `tool.img2pdf` | img2pdf | executable: `img2pdf` | experimental | deterministic | local | +| `tool.pandoc` | Pandoc | executable: `pandoc` | required | configuration_dependent | local | +| `tool.tectonic` | Tectonic | executable: `tectonic` | optional | configuration_dependent | network_optional | +| `tool.wkhtmltopdf` | wkhtmltopdf | executable: `wkhtmltopdf` | experimental | configuration_dependent | local | +| `tool.zip` | Info-ZIP compatible zip | executable: `zip` | experimental | configuration_dependent | local | + +## Capability matrix + +| Capability ID | Provider ID | +| --- | --- | +| `audio.convert` | `tool.ffmpeg` | +| `image.convert` | `tool.ffmpeg` | +| `media.convert` | `tool.ffmpeg` | +| `video.convert` | `tool.ffmpeg` | +| `pdf.process` | `tool.ghostscript` | +| `tiff.aggregate.press_pdf` | `tool.ghostscript` | +| `image.aggregate.pdf` | `tool.img2pdf` | +| `document.convert` | `tool.pandoc` | +| `document.generate` | `tool.pandoc` | +| `latex.compile` | `tool.tectonic` | +| `pdf.typeset` | `tool.tectonic` | +| `html.render.pdf` | `tool.wkhtmltopdf` | +| `archive.zip.create` | `tool.zip` | +| `comic.cbz.create` | `tool.zip` | + +## Runtime inspection + +Use the CLI to inspect the live host using the same model: + +```text +renderflow tools list +renderflow tools inspect tool.ffmpeg +renderflow capabilities +renderflow doctor +``` + +Add `--format json` or `--format yaml` to the tools/capabilities commands +when machine-readable evidence is needed. diff --git a/mkdocs.yml b/mkdocs.yml index c545d92..64f0967 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ nav: - User Guide: - Configuration: user-guide/configuration.md - Supported Formats: user-guide/supported-formats.md + - Tool Registry: user-guide/tool-registry.md - Pipelines: user-guide/pipelines.md - Transforms: user-guide/transforms.md - Optimization: user-guide/optimization.md @@ -117,6 +118,7 @@ nav: - graph: cli-reference/graph.md - plugin: cli-reference/plugin.md - ai: cli-reference/ai.md + - tools & capabilities: cli-reference/tools.md - Transform Reference: - Overview: transform-reference/index.md - Emoji: transform-reference/emoji.md diff --git a/scripts/generate_tool_registry_doc.py b/scripts/generate_tool_registry_doc.py new file mode 100755 index 0000000..67643c4 --- /dev/null +++ b/scripts/generate_tool_registry_doc.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = ROOT / "crates" / "renderflow-core" / "data" / "tool-registry.yaml" +DOC_PATH = ROOT / "docs" / "user-guide" / "tool-registry.md" + + +def render_table(headers: list[str], rows: list[list[str]]) -> list[str]: + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + for row in rows: + lines.append("| " + " | ".join(row) + " |") + return lines + + +def discovery_label(discovery: dict[str, object]) -> str: + kind = str(discovery.get("kind", "unknown")) + if kind == "executable": + candidates = discovery.get("candidates", []) + return "executable: " + ", ".join(f"`{value}`" for value in candidates) + if kind == "runtime_service": + return f"service: `{discovery.get('service', '-')}`" + return kind + + +def main() -> None: + registry = yaml.safe_load(REGISTRY_PATH.read_text(encoding="utf-8")) + schema = registry["schema"] + tools = sorted(registry.get("tools", []), key=lambda tool: tool["id"]) + + tool_rows: list[list[str]] = [] + capability_rows: list[list[str]] = [] + for tool in tools: + tool_rows.append( + [ + f"`{tool['id']}`", + str(tool["name"]), + discovery_label(tool["discovery"]), + str(tool.get("support_tier", "optional")), + str(tool.get("determinism", "-")), + str(tool.get("locality", "-")), + ] + ) + for capability in sorted(tool.get("capabilities", [])): + capability_rows.append([f"`{capability}`", f"`{tool['id']}`"]) + + content = [ + "# Tool Registry", + "", + "!!! info", + " This page is generated from the canonical built-in catalog at", + " `crates/renderflow-core/data/tool-registry.yaml` by", + " `scripts/generate_tool_registry_doc.py`. Do not edit it by hand.", + "", + f"Registry schema: `{schema}`", + "", + "The built-in catalog defines stable provider IDs, discovery/version probes,", + "platform constraints, capability IDs, determinism/locality/fidelity metadata,", + "runtime requirements, fallback relationships, and distribution/licensing notes.", + "YAML command transforms and plugins can register additional runtime providers", + "without editing this built-in catalog.", + "", + "## Built-in providers", + "", + *render_table( + ["Provider ID", "Name", "Discovery", "Tier", "Determinism", "Locality"], + tool_rows, + ), + "", + "## Capability matrix", + "", + *render_table(["Capability ID", "Provider ID"], capability_rows), + "", + "## Runtime inspection", + "", + "Use the CLI to inspect the live host using the same model:", + "", + "```text", + "renderflow tools list", + "renderflow tools inspect tool.ffmpeg", + "renderflow capabilities", + "renderflow doctor", + "```", + "", + "Add `--format json` or `--format yaml` to the tools/capabilities commands", + "when machine-readable evidence is needed.", + "", + ] + + DOC_PATH.write_text("\n".join(content), encoding="utf-8") + + +if __name__ == "__main__": + main()