diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 09da200..db16782 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,10 +8,13 @@ on: - 'docs/**' - 'mkdocs.yml' - 'scripts/generate_supported_formats_doc.py' - - 'src/audio/format.rs' - - 'src/graph/format.rs' - - 'src/image/format.rs' - - 'src/input_format.rs' + - 'scripts/generate_tool_registry_doc.py' + - 'crates/renderflow-core/data/tool-registry.yaml' + - 'crates/renderflow-core/src/audio/format.rs' + - 'crates/renderflow-core/src/graph/format.rs' + - 'crates/renderflow-core/src/image/format.rs' + - 'crates/renderflow-core/src/input_format.rs' + - 'crates/renderflow-core/src/toolchain.rs' push: branches: - main @@ -23,10 +26,13 @@ on: - 'docs/**' - 'mkdocs.yml' - 'scripts/generate_supported_formats_doc.py' - - 'src/audio/format.rs' - - 'src/graph/format.rs' - - 'src/image/format.rs' - - 'src/input_format.rs' + - 'scripts/generate_tool_registry_doc.py' + - 'crates/renderflow-core/data/tool-registry.yaml' + - 'crates/renderflow-core/src/audio/format.rs' + - 'crates/renderflow-core/src/graph/format.rs' + - 'crates/renderflow-core/src/image/format.rs' + - 'crates/renderflow-core/src/input_format.rs' + - 'crates/renderflow-core/src/toolchain.rs' workflow_dispatch: permissions: @@ -55,13 +61,15 @@ jobs: - name: Install MkDocs Material run: | python -m pip install --upgrade pip - pip install mkdocs-material mike + pip install mkdocs-material mike pyyaml - name: Regenerate generated docs - run: python scripts/generate_supported_formats_doc.py + run: | + python scripts/generate_supported_formats_doc.py + python scripts/generate_tool_registry_doc.py - name: Verify generated docs are committed - run: git diff --exit-code -- docs/user-guide/supported-formats.md + run: git diff --exit-code -- docs/user-guide/supported-formats.md docs/user-guide/tool-registry.md - name: Build docs run: mkdocs build --strict @@ -91,10 +99,12 @@ jobs: - name: Install MkDocs Material run: | python -m pip install --upgrade pip - pip install mkdocs-material mike + pip install mkdocs-material mike pyyaml - name: Regenerate generated docs - run: python scripts/generate_supported_formats_doc.py + run: | + python scripts/generate_supported_formats_doc.py + python scripts/generate_tool_registry_doc.py - name: Configure Pages uses: actions/configure-pages@v5 diff --git a/crates/renderflow-core/data/tool-registry.yaml b/crates/renderflow-core/data/tool-registry.yaml new file mode 100644 index 0000000..110bc54 --- /dev/null +++ b/crates/renderflow-core/data/tool-registry.yaml @@ -0,0 +1,161 @@ +schema: renderflow.tool-registry/v1 + +tools: + - id: tool.pandoc + name: Pandoc + discovery: + kind: executable + candidates: [pandoc] + version_args: [--version] + version: + min_inclusive: "2.0.0" + operating_systems: [linux, macos, windows] + capabilities: + - document.convert + - document.generate + input_media_types: + - text/markdown + - text/html + - text/x-rst + - application/epub+zip + - application/x-tex + output_media_types: + - text/html + - application/pdf + - application/vnd.openxmlformats-officedocument.wordprocessingml.document + - application/epub+zip + - application/x-tex + determinism: configuration_dependent + locality: local + fidelity: path_dependent + support_tier: required + license_notes: "See the Pandoc upstream license for redistribution terms." + distribution_notes: "Typically installed through the host package manager or official Pandoc release artifacts." + + - id: tool.tectonic + name: Tectonic + discovery: + kind: executable + candidates: [tectonic] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - pdf.typeset + - latex.compile + input_media_types: + - application/x-tex + - text/x-tex + output_media_types: + - application/pdf + determinism: configuration_dependent + locality: network_optional + fidelity: path_dependent + support_tier: optional + license_notes: "See the Tectonic upstream license for redistribution terms." + distribution_notes: "Optional unless a selected PDF path uses Tectonic." + + - id: tool.ffmpeg + name: FFmpeg + discovery: + kind: executable + candidates: [ffmpeg] + version_args: [--version] + version: + min_inclusive: "4.0.0" + operating_systems: [linux, macos, windows] + capabilities: + - media.convert + - audio.convert + - image.convert + - video.convert + input_media_types: + - audio/* + - image/* + - video/* + output_media_types: + - audio/* + - image/* + - video/* + determinism: configuration_dependent + locality: local + fidelity: path_dependent + support_tier: optional + license_notes: "FFmpeg licensing is build-dependent; consult the exact distributed build." + distribution_notes: "Used by Renderflow audio, image, and future video adapters." + + - id: tool.wkhtmltopdf + name: wkhtmltopdf + discovery: + kind: executable + candidates: [wkhtmltopdf] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - html.render.pdf + input_media_types: [text/html] + output_media_types: [application/pdf] + determinism: configuration_dependent + locality: local + fidelity: partial_loss + support_tier: experimental + license_notes: "Consult upstream wkhtmltopdf licensing and bundled Qt terms." + distribution_notes: "Alternative HTML-to-PDF provider; not required by the default document pipeline." + + - id: tool.zip + name: Info-ZIP compatible zip + discovery: + kind: executable + candidates: [zip] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - archive.zip.create + - comic.cbz.create + output_media_types: + - application/zip + - application/vnd.comicbook+zip + determinism: configuration_dependent + locality: local + fidelity: lossless + support_tier: experimental + license_notes: "License depends on the compatible zip implementation selected on PATH." + distribution_notes: "Used by command-backed CBZ aggregation when configured." + + - id: tool.img2pdf + name: img2pdf + discovery: + kind: executable + candidates: [img2pdf] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - image.aggregate.pdf + input_media_types: [image/*] + output_media_types: [application/pdf] + determinism: deterministic + locality: local + fidelity: lossless + support_tier: experimental + license_notes: "Consult the img2pdf upstream license for redistribution terms." + distribution_notes: "Used by lossless image-to-PDF aggregation when configured." + + - id: tool.ghostscript + name: Ghostscript + discovery: + kind: executable + candidates: [gs] + version_args: [--version] + operating_systems: [linux, macos, windows] + capabilities: + - pdf.process + - tiff.aggregate.press_pdf + input_media_types: + - image/tiff + - application/pdf + output_media_types: [application/pdf] + determinism: configuration_dependent + locality: local + fidelity: path_dependent + support_tier: experimental + license_notes: "Ghostscript is available under AGPL/commercial licensing; review distribution obligations." + distribution_notes: "Used by press-oriented TIFF/PDF command adapters when configured." diff --git a/crates/renderflow-core/src/adapters/command.rs b/crates/renderflow-core/src/adapters/command.rs index 0fb2917..9ab6c6b 100644 --- a/crates/renderflow-core/src/adapters/command.rs +++ b/crates/renderflow-core/src/adapters/command.rs @@ -50,7 +50,11 @@ pub fn run_command(program: &str, args: &[&str]) -> Result<()> { } result.ensure_success()?; - info!(program = program, duration_ms = result.duration_ms(), "Command completed successfully"); + info!( + program = program, + duration_ms = result.duration_ms(), + "Command completed successfully" + ); Ok(()) } diff --git a/crates/renderflow-core/src/app.rs b/crates/renderflow-core/src/app.rs index 67b5487..f7cef06 100644 --- a/crates/renderflow-core/src/app.rs +++ b/crates/renderflow-core/src/app.rs @@ -2,7 +2,7 @@ use anyhow::{bail, Result}; use clap::Parser; use tracing::info; -use crate::cli::{AiCommands, Cli, Commands, GraphCommands, PluginCommands}; +use crate::cli::{AiCommands, Cli, Commands, GraphCommands, PluginCommands, ToolCommands}; use crate::{commands, transforms}; /// Initialize logging for a Renderflow CLI run. @@ -133,6 +133,19 @@ pub fn run_cli(cli: Cli) -> Result<()> { optimization, } => commands::graph::run_stats(&config, target.as_deref(), optimization)?, }, + Some(Commands::Tools { subcommand }) => match subcommand { + ToolCommands::List { format, transforms } => { + commands::tools::run_list(transforms.as_deref(), &format)? + } + ToolCommands::Inspect { + id, + format, + transforms, + } => commands::tools::run_inspect(&id, transforms.as_deref(), &format)?, + }, + Some(Commands::Capabilities { format, transforms }) => { + commands::tools::run_capabilities(transforms.as_deref(), &format)? + } Some(Commands::Version) => commands::system::run_version(), Some(Commands::Env) => commands::system::run_env(), Some(Commands::Doctor { strict }) => commands::system::run_doctor(strict)?, diff --git a/crates/renderflow-core/src/artifact/cache.rs b/crates/renderflow-core/src/artifact/cache.rs index 386e1c3..75d31c5 100644 --- a/crates/renderflow-core/src/artifact/cache.rs +++ b/crates/renderflow-core/src/artifact/cache.rs @@ -96,7 +96,9 @@ pub(crate) fn save_artifact_cache(cache: &ArtifactCache, path: &Path) -> Result< let mut temporary = tempfile::NamedTempFile::new_in(parent) .context("Failed to create artifact cache temporary file")?; serde_json::to_writer(&mut temporary, cache).context("Failed to serialize artifact cache")?; - temporary.flush().context("Failed to flush artifact cache")?; + temporary + .flush() + .context("Failed to flush artifact cache")?; temporary .as_file() .sync_all() @@ -104,7 +106,12 @@ pub(crate) fn save_artifact_cache(cache: &ArtifactCache, path: &Path) -> Result< temporary .persist(path) .map_err(|error| error.error) - .with_context(|| format!("Failed to atomically save artifact cache '{}'", path.display()))?; + .with_context(|| { + format!( + "Failed to atomically save artifact cache '{}'", + path.display() + ) + })?; Ok(()) } diff --git a/crates/renderflow-core/src/artifact/store.rs b/crates/renderflow-core/src/artifact/store.rs index 6a390d5..0ea0d3a 100644 --- a/crates/renderflow-core/src/artifact/store.rs +++ b/crates/renderflow-core/src/artifact/store.rs @@ -216,7 +216,9 @@ impl ArtifactStore { artifact.id() ) })?; - temporary.flush().context("Failed to flush final artifact")?; + temporary + .flush() + .context("Failed to flush final artifact")?; temporary .as_file() .sync_all() @@ -293,9 +295,8 @@ mod tests { fn identical_payloads_share_content_storage() { let directory = tempfile::tempdir().unwrap(); let store = ArtifactStore::new(directory.path()).unwrap(); - let descriptor = || { - ArtifactDescriptor::for_format(Format::Png, ArtifactStorageClass::Intermediate) - }; + let descriptor = + || ArtifactDescriptor::for_format(Format::Png, ArtifactStorageClass::Intermediate); let first = store.put_bytes(b"\x89PNG\x00", descriptor()).unwrap(); let second = store.put_bytes(b"\x89PNG\x00", descriptor()).unwrap(); assert_eq!(first.id(), second.id()); diff --git a/crates/renderflow-core/src/cli.rs b/crates/renderflow-core/src/cli.rs index 440d58c..76a676b 100644 --- a/crates/renderflow-core/src/cli.rs +++ b/crates/renderflow-core/src/cli.rs @@ -27,6 +27,9 @@ use crate::optimization::OptimizationMode; renderflow version Print the installed version\n \ renderflow env Print installation environment details\n \ renderflow doctor Run installation diagnostics\n \ + renderflow tools list List runtime tool providers\n \ + renderflow tools inspect tool.ffmpeg Inspect one runtime provider\n \ + renderflow capabilities List provider capability IDs\n \ renderflow my-project.yaml Shorthand: run build on the given config" )] pub struct Cli { @@ -198,6 +201,23 @@ pub enum Commands { subcommand: GraphCommands, }, + /// Inspect the runtime external-tool/provider registry. + #[command(subcommand_required = true, arg_required_else_help = true)] + Tools { + #[command(subcommand)] + subcommand: ToolCommands, + }, + + /// List stable provider capability IDs and their implementations. + Capabilities { + /// Output format: text (default), json, or yaml. + #[arg(long, default_value = "text", value_name = "FORMAT")] + format: String, + /// Optional transform YAML whose dynamic providers should be included. + #[arg(long, value_name = "FILE")] + transforms: Option, + }, + /// Print the installed Renderflow version Version, @@ -238,6 +258,32 @@ pub enum PluginCommands { Doctor, } +/// Subcommands for `renderflow tools`. +#[derive(Subcommand)] +pub enum ToolCommands { + /// List registered providers and live availability/version state. + List { + /// Output format: text (default), json, or yaml. + #[arg(long, default_value = "text", value_name = "FORMAT")] + format: String, + /// Optional transform YAML whose dynamic providers should be included. + #[arg(long, value_name = "FILE")] + transforms: Option, + }, + + /// Inspect one stable provider ID. + Inspect { + /// Stable provider/tool identifier, for example `tool.ffmpeg`. + id: String, + /// Output format: text (default), json, or yaml. + #[arg(long, default_value = "text", value_name = "FORMAT")] + format: String, + /// Optional transform YAML whose dynamic providers should be included. + #[arg(long, value_name = "FILE")] + transforms: Option, + }, +} + /// Subcommands for `renderflow ai`. #[derive(Subcommand)] pub enum AiCommands { diff --git a/crates/renderflow-core/src/commands/graph.rs b/crates/renderflow-core/src/commands/graph.rs index 447d4cc..c2fa6de 100644 --- a/crates/renderflow-core/src/commands/graph.rs +++ b/crates/renderflow-core/src/commands/graph.rs @@ -8,13 +8,14 @@ use crate::graph::execution_plan::ExecutionPlan; use crate::graph::renderers::renderer_for; use crate::graph::{Format, MultiTargetDag}; use crate::optimization::OptimizationMode; -use crate::transforms::yaml_loader::build_graph_and_executor_from_yaml; +use crate::toolchain::filter_graph_for_current_toolchain; +use crate::transforms::yaml_loader::build_graph_executor_and_tools_from_yaml; // ── helpers ───────────────────────────────────────────────────────────────── /// Load the config, build the transform graph, compute the DAG, and return an /// [`ExecutionPlan`]. -fn load_plan( +pub(crate) fn load_plan( config_path: &str, target: Option<&str>, optimization: Option, @@ -29,8 +30,14 @@ fn load_plan( ) })?; - let (graph, _executor) = build_graph_and_executor_from_yaml(transforms_path)?; - info!("Loaded transform graph from '{}'", transforms_path); + let (raw_graph, _executor, tool_registry) = + build_graph_executor_and_tools_from_yaml(transforms_path)?; + let (graph, tool_inventory, tool_context) = + filter_graph_for_current_toolchain(&raw_graph, &tool_registry); + info!( + "Loaded tool-aware transform graph from '{}'", + transforms_path + ); let opt_mode = optimization.unwrap_or(config.optimization); @@ -61,12 +68,18 @@ fn load_plan( .ok_or_else(|| { anyhow::anyhow!( "Could not build an execution plan: one or more target formats \ - are not reachable from '{}' in the transform graph", - source_format + are not reachable from '{}' after provider availability filtering. Blocked providers: {}", + source_format, + tool_inventory.blocked_summaries().join("; ") ) })?; - let plan = ExecutionPlan::from_dag(&dag, source_format, &targets, opt_mode); + let mut plan = ExecutionPlan::from_dag(&dag, source_format, &targets, opt_mode); + let toolchain = tool_registry.fingerprint_for_dag(&tool_inventory, &dag, &tool_context)?; + plan.attach_toolchain(toolchain); + for diagnostic in tool_inventory.blocked_summaries() { + plan.add_tool_diagnostic(format!("Provider excluded from planning: {diagnostic}")); + } Ok((plan, targets)) } diff --git a/crates/renderflow-core/src/commands/graph_build.rs b/crates/renderflow-core/src/commands/graph_build.rs index 14157df..b4ad2b9 100644 --- a/crates/renderflow-core/src/commands/graph_build.rs +++ b/crates/renderflow-core/src/commands/graph_build.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{fs, path::Path}; use anyhow::{Context, Result}; use tracing::{debug, info}; @@ -8,7 +8,8 @@ use crate::config::load_config_for_graph; use crate::files::ensure_output_dir; use crate::graph::Format; use crate::optimization::OptimizationMode; -use crate::transforms::yaml_loader::build_graph_and_executor_from_yaml; +use crate::toolchain::filter_graph_for_current_toolchain; +use crate::transforms::yaml_loader::build_graph_executor_and_tools_from_yaml; /// Run graph-based execution targeting a single output format. pub fn run_target( @@ -46,7 +47,7 @@ fn run_impl( optimization: Option, ) -> Result<()> { if dry_run { - info!("Dry-run mode enabled — no files will be created and no commands will be executed"); + info!("Dry-run mode enabled — no files or transform commands will be produced; bounded tool probes may run for planning"); } info!("Running graph-based build pipeline"); @@ -60,8 +61,14 @@ fn run_impl( ) })?; - let (graph, executor) = build_graph_and_executor_from_yaml(transforms_path)?; - info!("Loaded transform graph from '{}'", transforms_path); + let (raw_graph, executor, tool_registry) = + build_graph_executor_and_tools_from_yaml(transforms_path)?; + let (graph, tool_inventory, tool_context) = + filter_graph_for_current_toolchain(&raw_graph, &tool_registry); + info!( + "Loaded tool-aware transform graph from '{}'", + transforms_path + ); let opt_mode = optimization.unwrap_or(config.optimization); info!(optimization = %opt_mode, "Using optimization mode"); @@ -101,11 +108,15 @@ fn run_impl( .ok_or_else(|| { anyhow::anyhow!( "Could not build an execution plan: one or more target formats \ - are not reachable from '{}' in the transform graph", - source_format + are not reachable from '{}' after provider availability filtering. Blocked providers: {}", + source_format, + tool_inventory.blocked_summaries().join("; ") ) })?; + let toolchain = tool_registry.fingerprint_for_dag(&tool_inventory, &dag, &tool_context)?; + info!(fingerprint = %toolchain.fingerprint, providers = toolchain.selected_tools.len(), "Resolved execution toolchain"); + debug!("Execution plan (DAG tree):\n{}", dag.to_tree(source_format)); let input_stem = Path::new(&config.input) @@ -139,7 +150,14 @@ fn run_impl( .unwrap_or_else(|| Path::new(".")); let state_dir = state_parent.join(".renderflow"); let artifact_store = ArtifactStore::new(state_dir.join("artifacts"))?; - let executor = executor.with_cache(state_dir.join("dag-cache.json")); + fs::create_dir_all(&state_dir)?; + fs::write( + state_dir.join("toolchain.json"), + serde_json::to_vec_pretty(&toolchain)?, + )?; + let executor = executor + .with_cache(state_dir.join("dag-cache.json")) + .with_toolchain_fingerprint(toolchain.fingerprint.clone()); let source_artifact = artifact_store.import_path( &config.input, diff --git a/crates/renderflow-core/src/commands/mod.rs b/crates/renderflow-core/src/commands/mod.rs index f81d487..ba1e8a2 100644 --- a/crates/renderflow-core/src/commands/mod.rs +++ b/crates/renderflow-core/src/commands/mod.rs @@ -6,4 +6,5 @@ pub mod graph_build; pub mod inspect; pub mod plugin; pub mod system; +pub mod tools; pub mod watch; diff --git a/crates/renderflow-core/src/commands/plugin.rs b/crates/renderflow-core/src/commands/plugin.rs index 8cfc979..a75fff3 100644 --- a/crates/renderflow-core/src/commands/plugin.rs +++ b/crates/renderflow-core/src/commands/plugin.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use crate::process::ProcessExecutor; +use crate::toolchain::{CapabilityId, ToolRegistry}; use crate::transforms::plugin::{PluginCapabilities, PluginMetadata, PluginRegistry}; // ── list ────────────────────────────────────────────────────────────────────── @@ -173,7 +173,22 @@ pub fn run_doctor(registry: &PluginRegistry) -> Result<()> { /// Return `true` when `tool` can be found and successfully version-probed. fn tool_is_available(tool: &str) -> bool { - ProcessExecutor::new().probe_version(tool).is_available() + let mut registry = ToolRegistry::builtins(); + let id = registry.canonical_id_for_executable(tool); + if !registry.contains(&id) { + let capability = CapabilityId::new("plugin.required_tool") + .expect("static capability identifier is valid"); + if registry + .ensure_command_provider(id.clone(), tool.to_string(), capability) + .is_err() + { + return false; + } + } + registry + .assess_ids_current([id.as_str()]) + .get(id.as_str()) + .is_some_and(|availability| availability.is_available()) } #[cfg(test)] diff --git a/crates/renderflow-core/src/commands/system.rs b/crates/renderflow-core/src/commands/system.rs index 9ce4c87..aaebdd9 100644 --- a/crates/renderflow-core/src/commands/system.rs +++ b/crates/renderflow-core/src/commands/system.rs @@ -1,43 +1,7 @@ use anyhow::{bail, Result}; use std::{env, path::PathBuf}; -use crate::process::{ProcessExecutor, ToolProbeStatus}; - -struct ToolCheck { - name: &'static str, - required: bool, -} - -// `pandoc` is required for core document rendering, while `tectonic` (PDF) -// and `ffmpeg` (media conversions) are optional unless those outputs are used. -const TOOL_CHECKS: [ToolCheck; 3] = [ - ToolCheck { - name: "pandoc", - required: true, - }, - ToolCheck { - name: "tectonic", - required: false, - }, - ToolCheck { - name: "ffmpeg", - required: false, - }, -]; - -fn probe_tool_version(name: &str) -> Result { - let probe = ProcessExecutor::new().probe_version(name); - match probe.status { - ToolProbeStatus::Available => Ok(probe - .version_line - .unwrap_or_else(|| "available".to_string())), - ToolProbeStatus::Missing => Err(format!("missing ({name} not found in PATH)")), - ToolProbeStatus::TimedOut => Err(format!("installed but version probe timed out ({name} --version)")), - ToolProbeStatus::Failed => Err(probe - .diagnostic - .unwrap_or_else(|| format!("installed but failed to execute ({name} --version)"))), - } -} +use crate::toolchain::{ToolRegistry, ToolSupportTier}; pub fn run_version() { println!("renderflow {}", env!("CARGO_PKG_VERSION")); @@ -58,36 +22,49 @@ pub fn run_env() { } pub fn run_doctor(strict: bool) -> Result<()> { + let registry = ToolRegistry::builtins(); + let inventory = registry.assess_all_current(); + println!("Renderflow Doctor"); println!("-----------------"); println!("renderflow: {}", env!("CARGO_PKG_VERSION")); println!("platform: {} {}", env::consts::OS, env::consts::ARCH); + println!("tool registry: {}", registry.schema()); + println!(); - let mut missing = 0usize; - - for check in TOOL_CHECKS { - match probe_tool_version(check.name) { - Ok(version) => println!("[ok] {}: {version}", check.name), - Err(reason) => { - if check.required { - missing += 1; - println!("[missing|required] {}: {reason}", check.name); - } else { - println!("[missing|optional] {}: {reason}", check.name); - } - } + let mut required_failures = 0usize; + for tool in &inventory.tools { + if tool.support_tier == ToolSupportTier::Required && !tool.is_available() { + required_failures += 1; } + let detail = tool + .version_line + .as_deref() + .or(tool.diagnostic.as_deref()) + .unwrap_or("no additional detail"); + println!( + "[{}|{}] {}: {}", + tool.status.as_str(), + tool.support_tier.as_str(), + tool.id, + detail + ); } - if strict && missing > 0 { - bail!("doctor found {missing} required dependency issue(s)"); + if strict && required_failures > 0 { + bail!("doctor found {required_failures} required toolchain issue(s)"); } - if missing == 0 { - println!("Doctor completed: required dependencies look healthy."); + if required_failures == 0 { + println!( + " +Doctor completed: required toolchain providers look healthy." + ); } else { - println!("Doctor completed with warnings. Install missing required tools and retry."); + println!( + " +Doctor completed with required-provider warnings." + ); } - Ok(()) } diff --git a/crates/renderflow-core/src/commands/tools.rs b/crates/renderflow-core/src/commands/tools.rs new file mode 100644 index 0000000..c7bd510 --- /dev/null +++ b/crates/renderflow-core/src/commands/tools.rs @@ -0,0 +1,184 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, Result}; +use serde::Serialize; + +use crate::toolchain::{ToolAvailability, ToolDescriptor, ToolRegistry}; +use crate::transforms::yaml_loader::load_tool_registry_from_yaml; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StructuredFormat { + Text, + Json, + Yaml, +} + +impl StructuredFormat { + fn parse(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "text" => Ok(Self::Text), + "json" => Ok(Self::Json), + "yaml" | "yml" => Ok(Self::Yaml), + other => { + anyhow::bail!("unknown tools output format '{other}'; supported: text, json, yaml") + } + } + } +} + +fn load_registry(transforms: Option<&str>) -> Result { + match transforms { + Some(path) => load_tool_registry_from_yaml(path) + .with_context(|| format!("failed to load tool providers from '{path}'")), + None => Ok(ToolRegistry::builtins()), + } +} + +fn emit_serialized(value: &T, format: StructuredFormat) -> Result<()> { + match format { + StructuredFormat::Text => anyhow::bail!("text output requires a dedicated renderer"), + StructuredFormat::Json => { + println!("{}", serde_json::to_string_pretty(value)?); + } + StructuredFormat::Yaml => { + print!("{}", serde_yaml_ng::to_string(value)?); + } + } + Ok(()) +} + +pub fn run_list(transforms: Option<&str>, format: &str) -> Result<()> { + let registry = load_registry(transforms)?; + let inventory = registry.assess_all_current(); + let format = StructuredFormat::parse(format)?; + + if format != StructuredFormat::Text { + return emit_serialized(&inventory, format); + } + + println!("Renderflow Tools"); + println!("================"); + println!("registry: {}", registry.schema()); + println!(); + for tool in &inventory.tools { + let executable = tool.selected_executable.as_deref().unwrap_or("-"); + let version = tool + .normalized_version + .as_deref() + .or(tool.version_line.as_deref()) + .unwrap_or("-"); + println!( + "{:<24} {:<24} {:<12} {:<16} {}", + tool.id, + tool.status.as_str(), + tool.support_tier.as_str(), + executable, + version + ); + } + Ok(()) +} + +#[derive(Serialize)] +struct ToolInspection<'a> { + descriptor: &'a ToolDescriptor, + availability: &'a ToolAvailability, +} + +pub fn run_inspect(id: &str, transforms: Option<&str>, format: &str) -> Result<()> { + let registry = load_registry(transforms)?; + let descriptor = registry + .get(id) + .ok_or_else(|| anyhow::anyhow!("tool provider '{id}' is not registered"))?; + let inventory = registry.assess_ids_current([id]); + let availability = inventory + .get(id) + .expect("requested registered tool is present in its inventory"); + let format = StructuredFormat::parse(format)?; + + if format != StructuredFormat::Text { + return emit_serialized( + &ToolInspection { + descriptor, + availability, + }, + format, + ); + } + + println!("Tool: {}", descriptor.id); + println!("Name: {}", descriptor.name); + println!("Status: {}", availability.status.as_str()); + println!("Support tier: {}", descriptor.support_tier.as_str()); + println!( + "Executable: {}", + availability.selected_executable.as_deref().unwrap_or("-") + ); + println!( + "Version: {}", + availability + .version_line + .as_deref() + .unwrap_or("not captured") + ); + println!("Discovery: {:?}", descriptor.discovery); + println!("Determinism: {:?}", descriptor.determinism); + println!("Locality: {:?}", descriptor.locality); + println!("Fidelity: {:?}", descriptor.fidelity); + println!("Capabilities:"); + for capability in &descriptor.capabilities { + println!(" - {capability}"); + } + if !descriptor.fallbacks.is_empty() { + println!("Fallbacks:"); + for fallback in &descriptor.fallbacks { + println!(" - {fallback}"); + } + } + if let Some(diagnostic) = &availability.diagnostic { + println!("Diagnostic: {diagnostic}"); + } + Ok(()) +} + +pub fn run_capabilities(transforms: Option<&str>, format: &str) -> Result<()> { + let registry = load_registry(transforms)?; + let capabilities: BTreeMap> = registry.capabilities(); + let format = StructuredFormat::parse(format)?; + + if format != StructuredFormat::Text { + return emit_serialized(&capabilities, format); + } + + println!("Renderflow Capabilities"); + println!("======================="); + for (capability, providers) in capabilities { + println!("{capability}"); + for provider in providers { + println!(" - {provider}"); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn structured_format_rejects_unknown_value() { + assert!(StructuredFormat::parse("toml").is_err()); + } + + #[test] + fn built_in_capability_projection_is_not_empty() { + let registry = ToolRegistry::builtins(); + assert!(!registry.capabilities().is_empty()); + } + + #[test] + fn inventory_type_remains_structured_for_cli_serialization() { + let inventory = crate::toolchain::ToolInventory { tools: Vec::new() }; + assert!(serde_json::to_string(&inventory).is_ok()); + } +} diff --git a/crates/renderflow-core/src/deps.rs b/crates/renderflow-core/src/deps.rs index ba95f64..d12985e 100644 --- a/crates/renderflow-core/src/deps.rs +++ b/crates/renderflow-core/src/deps.rs @@ -1,12 +1,27 @@ use anyhow::Result; use crate::error::RenderError; -use crate::process::ProcessExecutor; +use crate::toolchain::{CapabilityId, ToolRegistry}; /// Check whether a tool is available in the system PATH using the canonical /// bounded version-probe path. fn tool_available(name: &str) -> bool { - ProcessExecutor::new().probe_version(name).is_available() + let mut registry = ToolRegistry::builtins(); + let id = registry.canonical_id_for_executable(name); + if !registry.contains(&id) { + let capability = + CapabilityId::new("probe.availability").expect("static capability identifier is valid"); + if registry + .ensure_command_provider(id.clone(), name.to_string(), capability) + .is_err() + { + return false; + } + } + registry + .assess_ids_current([id.as_str()]) + .get(id.as_str()) + .is_some_and(|tool| tool.is_available()) } /// Verify that `pandoc` is installed and available in PATH. diff --git a/crates/renderflow-core/src/graph/capability.rs b/crates/renderflow-core/src/graph/capability.rs index 20ce275..681acaa 100644 --- a/crates/renderflow-core/src/graph/capability.rs +++ b/crates/renderflow-core/src/graph/capability.rs @@ -183,6 +183,15 @@ impl ExternalTool { pub fn as_str(self) -> &'static str { self.executable_name() } + + /// Return the stable provider ID used by the runtime tool registry. + pub fn stable_id(self) -> &'static str { + match self { + Self::Pandoc => "tool.pandoc", + Self::Tectonic => "tool.tectonic", + Self::Ffmpeg => "tool.ffmpeg", + } + } } impl std::fmt::Display for ExternalTool { diff --git a/crates/renderflow-core/src/graph/dag_executor.rs b/crates/renderflow-core/src/graph/dag_executor.rs index d2d7fc5..6b9d847 100644 --- a/crates/renderflow-core/src/graph/dag_executor.rs +++ b/crates/renderflow-core/src/graph/dag_executor.rs @@ -8,9 +8,9 @@ use tracing::{debug, warn}; use super::{Format, MultiTargetDag, TransformEdge}; use crate::artifact::{ - compute_artifact_node_hash, load_artifact_cache, save_artifact_cache, Artifact, - ArtifactCache, ArtifactCollection, ArtifactDescriptor, ArtifactStorageClass, ArtifactStore, - ArtifactTransform, TextTransformAdapter, + compute_artifact_node_hash, load_artifact_cache, save_artifact_cache, Artifact, ArtifactCache, + ArtifactCollection, ArtifactDescriptor, ArtifactStorageClass, ArtifactStore, ArtifactTransform, + TextTransformAdapter, }; use crate::transforms::aggregation::AggregationTransform; use crate::transforms::Transform; @@ -27,6 +27,8 @@ pub struct DagExecutor { aggregation_transforms: HashMap<(Format, Format), Arc>, /// Optional artifact-native DAG cache path. cache_path: Option, + /// Selected-provider fingerprint used to reject incompatible cache entries. + toolchain_fingerprint: Option, } impl DagExecutor { @@ -36,6 +38,7 @@ impl DagExecutor { single_transforms: HashMap::new(), aggregation_transforms: HashMap::new(), cache_path: None, + toolchain_fingerprint: None, } } @@ -48,6 +51,12 @@ impl DagExecutor { self } + /// Attach a selected-provider fingerprint to cache compatibility. + pub fn with_toolchain_fingerprint(mut self, fingerprint: impl Into) -> Self { + self.toolchain_fingerprint = Some(fingerprint.into()); + self + } + /// Register an existing UTF-8 text transform through the compatibility adapter. pub fn register_single( &mut self, @@ -55,10 +64,8 @@ impl DagExecutor { to: Format, transform: Arc, ) -> &mut Self { - self.single_transforms.insert( - (from, to), - Arc::new(TextTransformAdapter::new(transform)), - ); + self.single_transforms + .insert((from, to), Arc::new(TextTransformAdapter::new(transform))); self } @@ -302,9 +309,12 @@ impl DagExecutor { edge.to ) })?; - let cache_identity = transform.cache_identity(); - let cache_key = - compute_artifact_node_hash(input, edge.from, edge.to, &cache_identity); + let mut cache_identity = transform.cache_identity(); + if let Some(fingerprint) = &self.toolchain_fingerprint { + cache_identity.push_str("\0toolchain="); + cache_identity.push_str(fingerprint); + } + let cache_key = compute_artifact_node_hash(input, edge.from, edge.to, &cache_identity); if let Some(cache_mutex) = cache { if let Ok(guard) = cache_mutex.lock() { @@ -490,11 +500,8 @@ mod tests { let mut reader = store.open(input)?; store.put_reader( &mut reader, - ArtifactDescriptor::for_format( - output_format, - ArtifactStorageClass::Intermediate, - ) - .with_source(input.id().clone()), + ArtifactDescriptor::for_format(output_format, ArtifactStorageClass::Intermediate) + .with_source(input.id().clone()), ) } } @@ -522,11 +529,8 @@ mod tests { let mut reader = store.open(input)?; store.put_reader( &mut reader, - ArtifactDescriptor::for_format( - output_format, - ArtifactStorageClass::Intermediate, - ) - .with_source(input.id().clone()), + ArtifactDescriptor::for_format(output_format, ArtifactStorageClass::Intermediate) + .with_source(input.id().clone()), ) } } @@ -563,11 +567,7 @@ mod tests { fn one_edge(from: Format, to: Format, input_kind: InputKind) -> MultiTargetDag { let mut graph = TransformGraph::new(); graph.add_transform(TransformEdge::with_input_kind( - from, - to, - 1.0, - 1.0, - input_kind, + from, to, 1.0, 1.0, input_kind, )); graph .build_multi_target_dag(from, &[to]) @@ -602,11 +602,7 @@ mod tests { ) .unwrap(); let mut executor = DagExecutor::new(); - executor.register_artifact( - Format::Png, - Format::Webp, - Arc::new(BinaryCopyTransform), - ); + executor.register_artifact(Format::Png, Format::Webp, Arc::new(BinaryCopyTransform)); let results = executor .execute_artifact(&dag, Format::Png, source.clone(), &store) @@ -635,11 +631,7 @@ mod tests { ) .unwrap(); let mut executor = DagExecutor::new(); - executor.register_aggregation( - Format::Png, - Format::Pdf, - Arc::new(OrderedJoinAggregation), - ); + executor.register_aggregation(Format::Png, Format::Pdf, Arc::new(OrderedJoinAggregation)); let results = executor .execute_artifacts( @@ -651,10 +643,7 @@ mod tests { .unwrap(); let output = results[&Format::Pdf].clone().into_one().unwrap(); assert_eq!(store.read_bytes(&output).unwrap(), b"page-one|page-two"); - assert_eq!( - output.sources(), - &[first.id().clone(), second.id().clone()] - ); + assert_eq!(output.sources(), &[first.id().clone(), second.id().clone()]); } #[test] @@ -700,19 +689,11 @@ mod tests { ) .unwrap(); let mut executor = DagExecutor::new(); - executor.register_aggregation( - Format::Png, - Format::Pdf, - Arc::new(FailingAggregation), - ); + executor.register_aggregation(Format::Png, Format::Pdf, Arc::new(FailingAggregation)); let before = count_artifact_objects(&store); - let result = executor.execute_artifacts( - &dag, - Format::Png, - ArtifactCollection::one(source), - &store, - ); + let result = + executor.execute_artifacts(&dag, Format::Png, ArtifactCollection::one(source), &store); assert!(result.is_err()); assert_eq!(before, count_artifact_objects(&store)); } diff --git a/crates/renderflow-core/src/graph/definition.rs b/crates/renderflow-core/src/graph/definition.rs index ab5ce21..b4f56d4 100644 --- a/crates/renderflow-core/src/graph/definition.rs +++ b/crates/renderflow-core/src/graph/definition.rs @@ -1,4 +1,5 @@ use super::{Format, InputKind, TransformEdge}; +use crate::toolchain::{canonical_tool_id_for_hint, transform_capability_id}; /// A pluggable definition of a format-to-format transformation. /// @@ -103,7 +104,10 @@ impl TransformDefinition { /// Convert this definition into a [`TransformEdge`] for use in a /// [`TransformGraph`](super::TransformGraph). pub fn to_edge(&self) -> TransformEdge { + let provider = canonical_tool_id_for_hint(&self.label); + let capability = transform_capability_id(self.from, self.to); TransformEdge::with_input_kind(self.from, self.to, self.cost, self.quality, self.input_kind) + .with_provider(provider.to_string(), capability.to_string()) } } diff --git a/crates/renderflow-core/src/graph/execution_plan.rs b/crates/renderflow-core/src/graph/execution_plan.rs index c478ece..8938306 100644 --- a/crates/renderflow-core/src/graph/execution_plan.rs +++ b/crates/renderflow-core/src/graph/execution_plan.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use super::{Format, InputKind, MultiTargetDag, TransformEdge}; use crate::optimization::OptimizationMode; +use crate::toolchain::ToolchainSnapshot; // ── Node types ───────────────────────────────────────────────────────────── @@ -60,6 +61,12 @@ pub struct PlanEdge { pub input_kind: String, /// Semantic classification of this edge. pub edge_type: EdgeType, + /// Stable provider/tool identifier selected for this edge. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_id: Option, + /// Stable capability identifier implemented by the selected provider. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capability_id: Option, } impl PlanEdge { @@ -82,6 +89,8 @@ impl PlanEdge { InputKind::Collection => "collection".to_string(), }, edge_type, + provider_id: e.provider_id.clone(), + capability_id: e.capability_id.clone(), } } } @@ -206,6 +215,9 @@ pub struct ExecutionPlan { pub metadata: ExecutionMetadata, /// Planner observations and explanations. pub diagnostics: Vec, + /// Reproducible evidence for providers selected by this exact plan. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub toolchain: Option, } impl ExecutionPlan { @@ -313,9 +325,35 @@ impl ExecutionPlan { waves, metadata, diagnostics, + toolchain: None, } } + /// Attach selected-provider evidence and a deterministic toolchain fingerprint. + pub fn attach_toolchain(&mut self, snapshot: ToolchainSnapshot) { + let selected = snapshot + .selected_tools + .iter() + .map(|tool| tool.id.to_string()) + .collect::>() + .join(", "); + self.diagnostics.push(PlanDiagnostic::info(format!( + "Toolchain fingerprint {} selected provider(s): {}.", + snapshot.fingerprint, + if selected.is_empty() { + "(none)" + } else { + &selected + } + ))); + self.toolchain = Some(snapshot); + } + + /// Surface an unavailable/unsupported provider observation in plan diagnostics. + pub fn add_tool_diagnostic(&mut self, message: impl Into) { + self.diagnostics.push(PlanDiagnostic::warning(message)); + } + // ── private helpers ──────────────────────────────────────────────────── /// Group edges into parallel execution waves. diff --git a/crates/renderflow-core/src/graph/mod.rs b/crates/renderflow-core/src/graph/mod.rs index 5308a47..4a30ec5 100644 --- a/crates/renderflow-core/src/graph/mod.rs +++ b/crates/renderflow-core/src/graph/mod.rs @@ -24,7 +24,7 @@ pub use transform_edge::TransformEdge; use crate::optimization::OptimizationMode; use petgraph::graph::{DiGraph, NodeIndex}; use petgraph::visit::EdgeRef; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// A directed graph of document format transformations. /// @@ -123,6 +123,34 @@ impl TransformGraph { .collect() } + /// Return stable provider IDs referenced by graph edges. + pub fn provider_ids(&self) -> Vec { + let mut ids: Vec = self + .graph + .edge_references() + .filter_map(|edge| edge.weight().provider_id.clone()) + .collect(); + ids.sort(); + ids.dedup(); + ids + } + + /// Clone the graph while excluding edges whose declared provider is unavailable. + /// Legacy edges without provider identity remain available for compatibility. + pub fn filtered_by_available_providers(&self, available: &HashSet) -> Self { + let mut filtered = Self::new(); + for edge in self.graph.edge_references().map(|edge| edge.weight()) { + if edge + .provider_id + .as_ref() + .is_none_or(|provider| available.contains(provider)) + { + filtered.add_transform(edge.clone()); + } + } + filtered + } + /// Return `true` when at least one direct transformation from `from` to /// `to` has been registered. pub fn has_transform(&self, from: Format, to: Format) -> bool { diff --git a/crates/renderflow-core/src/graph/renderers/text.rs b/crates/renderflow-core/src/graph/renderers/text.rs index 0f064a8..5bc7a5e 100644 --- a/crates/renderflow-core/src/graph/renderers/text.rs +++ b/crates/renderflow-core/src/graph/renderers/text.rs @@ -18,6 +18,9 @@ impl PlanRenderer for TextRenderer { let _ = writeln!(out, "Source: {}", plan.source); let _ = writeln!(out, "Targets: {}", plan.targets.join(", ")); let _ = writeln!(out, "Optimization: {}", plan.optimization); + if let Some(toolchain) = &plan.toolchain { + let _ = writeln!(out, "Toolchain: {}", toolchain.fingerprint); + } // ── nodes ────────────────────────────────────────────────────────── let _ = writeln!(out); @@ -36,13 +39,19 @@ impl PlanRenderer for TextRenderer { let _ = writeln!(out, "Edges ({}):", plan.metadata.total_edges); let max_from = plan.edges.iter().map(|e| e.from.len()).max().unwrap_or(0); for e in &plan.edges { + let provider = e + .provider_id + .as_deref() + .map(|value| format!(", provider: {value}")) + .unwrap_or_default(); let _ = writeln!( out, - " {:, + /// Stable machine-readable capability identifier for this edge, when known. + pub capability_id: Option, } impl TransformEdge { @@ -41,6 +45,8 @@ impl TransformEdge { cost, quality: quality.clamp(0.0, 1.0), input_kind: InputKind::Single, + provider_id: None, + capability_id: None, } } @@ -68,8 +74,21 @@ impl TransformEdge { cost, quality: quality.clamp(0.0, 1.0), input_kind, + provider_id: None, + capability_id: None, } } + + /// Attach stable provider and capability identity used by planning/evidence. + pub fn with_provider( + mut self, + provider_id: impl Into, + capability_id: impl Into, + ) -> Self { + self.provider_id = Some(provider_id.into()); + self.capability_id = Some(capability_id.into()); + self + } } #[cfg(test)] diff --git a/crates/renderflow-core/src/lib.rs b/crates/renderflow-core/src/lib.rs index 1228140..8884b8b 100644 --- a/crates/renderflow-core/src/lib.rs +++ b/crates/renderflow-core/src/lib.rs @@ -28,6 +28,7 @@ pub mod process; mod sdk; pub mod strategies; mod template; +pub mod toolchain; pub mod transforms; pub use sdk::{ diff --git a/crates/renderflow-core/src/process.rs b/crates/renderflow-core/src/process.rs index 74ba329..c0aad28 100644 --- a/crates/renderflow-core/src/process.rs +++ b/crates/renderflow-core/src/process.rs @@ -124,10 +124,7 @@ impl fmt::Debug for ProcessInput { match self { Self::Null => f.write_str("Null"), Self::Inherit => f.write_str("Inherit"), - Self::Bytes(bytes) => f - .debug_struct("Bytes") - .field("len", &bytes.len()) - .finish(), + Self::Bytes(bytes) => f.debug_struct("Bytes").field("len", &bytes.len()).finish(), } } } @@ -237,7 +234,8 @@ impl ProcessEnvironment { } pub fn allow_sensitive(mut self, name: impl Into) -> Self { - self.allow_sensitive.insert(normalize_env_name(&name.into())); + self.allow_sensitive + .insert(normalize_env_name(&name.into())); self } @@ -259,11 +257,7 @@ impl ProcessEnvironment { self } - pub fn set_sensitive( - mut self, - name: impl Into, - value: impl Into, - ) -> Self { + pub fn set_sensitive(mut self, name: impl Into, value: impl Into) -> Self { let name = name.into(); self.overrides.insert( normalize_env_name(&name), @@ -389,7 +383,10 @@ impl fmt::Debug for ProcessRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ProcessRequest") .field("executable", &self.executable) - .field("args", &safe_argument_display(&self.args, &Redactor::default())) + .field( + "args", + &safe_argument_display(&self.args, &Redactor::default()), + ) .field("invocation_kind", &self.invocation_kind) .field("working_directory", &self.working_directory) .field("stdin", &self.stdin) @@ -451,8 +448,11 @@ impl ProcessRequest { I: IntoIterator, S: Into, { - self.args - .extend(values.into_iter().map(|value| ProcessArgument::plain(value))); + self.args.extend( + values + .into_iter() + .map(|value| ProcessArgument::plain(value)), + ); self } @@ -682,26 +682,29 @@ impl ProcessExecutor { .is_some_and(ProcessCancellationToken::is_cancelled) { terminate_process_tree(&mut child, request.tree_mode).map_err(|error| { - ProcessError::Io(redactor.redact(&format!( - "failed to terminate cancelled process: {error}" - ))) + ProcessError::Io( + redactor.redact(&format!("failed to terminate cancelled process: {error}")), + ) })?; break ProcessTermination::Cancelled; } - if request.timeout.is_some_and(|timeout| started.elapsed() >= timeout) { + if request + .timeout + .is_some_and(|timeout| started.elapsed() >= timeout) + { terminate_process_tree(&mut child, request.tree_mode).map_err(|error| { - ProcessError::Io(redactor.redact(&format!( - "failed to terminate timed-out process: {error}" - ))) + ProcessError::Io( + redactor.redact(&format!("failed to terminate timed-out process: {error}")), + ) })?; break ProcessTermination::TimedOut; } match child.try_wait().map_err(|error| { - ProcessError::Io(redactor.redact(&format!( - "failed while waiting for process: {error}" - ))) + ProcessError::Io( + redactor.redact(&format!("failed while waiting for process: {error}")), + ) })? { Some(status) => break termination_from_status(status), None => thread::sleep(POLL_INTERVAL), @@ -715,9 +718,9 @@ impl ProcessExecutor { debug!(error = %redactor.redact(&error.to_string()), "stdin writer ended after process termination"); } Ok(Err(error)) => { - return Err(ProcessError::Io(redactor.redact(&format!( - "failed to write process stdin: {error}" - )))); + return Err(ProcessError::Io( + redactor.redact(&format!("failed to write process stdin: {error}")), + )); } Err(_) if !termination.is_success() => {} Err(_) => { @@ -777,8 +780,17 @@ impl ProcessExecutor { /// Probe ` --version` using the same bounded process policy. pub fn probe_version(&self, executable: &str) -> ToolProbeEvidence { + self.probe_version_with_args(executable, &["--version".to_string()]) + } + + /// Probe a tool version using an explicit argv declaration from the tool registry. + pub fn probe_version_with_args( + &self, + executable: &str, + version_args: &[String], + ) -> ToolProbeEvidence { let request = ProcessRequest::direct(executable) - .arg("--version") + .args(version_args.iter().cloned()) .timeout(PROBE_TIMEOUT) .capture_limit(PROBE_CAPTURE_LIMIT_BYTES); @@ -882,7 +894,8 @@ impl CapturedOutput { if self.truncated { format!( "{text}\n[capture truncated: retained {} of {} bytes]", - self.bytes.len(), self.total_bytes + self.bytes.len(), + self.total_bytes ) } else { text.to_string() @@ -960,7 +973,10 @@ impl ProcessResult { format!("Command `{}` exited with code {code}", self.command_display) } ProcessTermination::Signaled => { - format!("Command `{}` was terminated by a signal", self.command_display) + format!( + "Command `{}` was terminated by a signal", + self.command_display + ) } ProcessTermination::TimedOut => { format!("Command `{}` timed out", self.command_display) @@ -1093,12 +1109,18 @@ impl ProcessExpectedOutput { fn validate(&self, before: &OutputSnapshot) -> Option { let after = OutputSnapshot::capture(&self.path); if !after.exists { - return Some(format!("expected output '{}' was not produced", self.path.display())); + return Some(format!( + "expected output '{}' was not produced", + self.path.display() + )); } match self.kind { ExpectedOutputKind::Any => {} ExpectedOutputKind::File if !after.is_file => { - return Some(format!("expected output '{}' is not a file", self.path.display())); + return Some(format!( + "expected output '{}' is not a file", + self.path.display() + )); } ExpectedOutputKind::Directory if !after.is_directory => { return Some(format!( @@ -1109,7 +1131,10 @@ impl ProcessExpectedOutput { _ => {} } if self.require_non_empty && after.is_file && after.len == Some(0) { - return Some(format!("expected output '{}' is empty", self.path.display())); + return Some(format!( + "expected output '{}' is empty", + self.path.display() + )); } if self.require_change && &after == before { return Some(format!( @@ -1139,7 +1164,8 @@ impl Redactor { { self.secrets .extend(values.into_iter().filter(|value| !value.is_empty())); - self.secrets.sort_by_key(|value| std::cmp::Reverse(value.len())); + self.secrets + .sort_by_key(|value| std::cmp::Reverse(value.len())); self.secrets.dedup(); } @@ -1159,7 +1185,10 @@ struct BoundedBytes { truncated: bool, } -fn spawn_bounded_reader(reader: R, max_bytes: usize) -> thread::JoinHandle> +fn spawn_bounded_reader( + reader: R, + max_bytes: usize, +) -> thread::JoinHandle> where R: Read + Send + 'static, { @@ -1351,8 +1380,7 @@ pub(crate) fn is_explicit_shell_invocation(executable: &str, args: &[String]) -> .to_ascii_lowercase(); let is_shell = matches!( basename.as_str(), - "sh" - | "bash" + "sh" | "bash" | "zsh" | "dash" | "ksh" @@ -1542,8 +1570,7 @@ mod tests { let output = directory.path().join("missing.out"); let error = ProcessExecutor::new() .execute_checked( - ProcessRequest::direct("true") - .expect_output(ProcessExpectedOutput::file(&output)), + ProcessRequest::direct("true").expect_output(ProcessExpectedOutput::file(&output)), ) .unwrap_err(); assert!(error.to_string().contains("was not produced")); diff --git a/crates/renderflow-core/src/sdk.rs b/crates/renderflow-core/src/sdk.rs index f68ec39..90f0bd3 100644 --- a/crates/renderflow-core/src/sdk.rs +++ b/crates/renderflow-core/src/sdk.rs @@ -8,10 +8,10 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::commands; -use crate::config::{load_config, load_config_for_graph}; -use crate::graph::{ExecutionPlan, Format}; +use crate::config::load_config; +use crate::graph::ExecutionPlan; use crate::optimization::OptimizationMode; -use crate::transforms::yaml_loader::build_graph_and_executor_from_yaml; +use crate::toolchain::ToolchainSnapshot; #[derive(Debug, Error)] pub enum RenderflowError { @@ -172,6 +172,8 @@ pub struct ExecutionResult { pub reused_cached_outputs: Vec, pub skipped_transforms: Vec, pub diagnostics: DiagnosticReport, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub toolchain: Option, } #[derive(Default)] @@ -262,58 +264,17 @@ impl Engine { pub fn plan(&self, request: PlanRequest) -> Result { self.ensure_not_cancelled()?; self.emit(ProgressStage::Planning, "Constructing execution plan"); - - let config = load_config_for_graph(request.config_path.to_str().ok_or_else(|| { + let config_path = request.config_path.to_str().ok_or_else(|| { RenderflowError::Planning(anyhow::anyhow!("Config path contains non-UTF8 characters")) - })?) - .map_err(RenderflowError::Planning)?; - - let transforms_path = config.transforms.as_deref().ok_or_else(|| { - RenderflowError::Planning(anyhow::anyhow!( - "Planning requires a `transforms` key in the config file" - )) })?; - - let (graph, _executor) = build_graph_and_executor_from_yaml(transforms_path) - .map_err(RenderflowError::Planning)?; - - let opt_mode = request.optimization.unwrap_or(config.optimization); - let source_format: Format = config - .input_format() - .to_string() - .parse() - .map_err(|err| RenderflowError::Planning(anyhow::anyhow!("{}", err)))?; - - let targets: Vec = if let Some(target) = request.target { - vec![target - .parse::() - .map_err(|err| RenderflowError::Planning(anyhow::anyhow!("{}", err)))?] - } else { - let reachable = graph.reachable_from(source_format); - if reachable.is_empty() { - return Err(RenderflowError::Planning(anyhow::anyhow!( - "No output formats are reachable from '{}'", - source_format - ))); - } - reachable - }; - - let dag = graph - .build_multi_target_dag_with_mode(source_format, &targets, opt_mode) - .ok_or_else(|| { - RenderflowError::Planning(anyhow::anyhow!( - "Could not build an execution plan for one or more targets" - )) - })?; - + let (plan, _targets) = commands::graph::load_plan( + config_path, + request.target.as_deref(), + request.optimization, + ) + .map_err(RenderflowError::Planning)?; self.emit(ProgressStage::Completed, "Planning complete"); - Ok(ExecutionPlan::from_dag( - &dag, - source_format, - &targets, - opt_mode, - )) + Ok(plan) } pub fn execute(&self, request: ExecutionRequest) -> Result { @@ -326,6 +287,18 @@ impl Engine { let config = load_config(config_path).map_err(RenderflowError::Execution)?; + let toolchain = if request.target.is_some() || request.all_targets { + let (plan, _targets) = commands::graph::load_plan( + config_path, + request.target.as_deref(), + request.optimization, + ) + .map_err(RenderflowError::Execution)?; + plan.toolchain + } else { + None + }; + if let Some(target) = request.target.as_deref() { commands::graph_build::run_target( config_path, @@ -365,6 +338,7 @@ impl Engine { warnings: Vec::new(), recoverable_failures: Vec::new(), }, + toolchain, }) } } diff --git a/crates/renderflow-core/src/strategies/pdf.rs b/crates/renderflow-core/src/strategies/pdf.rs index 4807e9e..394c330 100644 --- a/crates/renderflow-core/src/strategies/pdf.rs +++ b/crates/renderflow-core/src/strategies/pdf.rs @@ -3,8 +3,8 @@ use std::path::Path; use tracing::info; use crate::adapters::command::run_command; -use crate::process::{ProcessExecutor, ToolProbeStatus}; use crate::strategies::{OutputStrategy, PandocArgs, RenderContext}; +use crate::toolchain::ToolRegistry; /// Renders a document to PDF format using pandoc with the tectonic PDF engine. pub struct PdfStrategy { @@ -23,26 +23,20 @@ impl PdfStrategy { /// Returns an error if the tectonic PDF engine is not installed or cannot /// be version-probed through the canonical process executor. fn check_tectonic() -> Result<()> { - let probe = ProcessExecutor::new().probe_version("tectonic"); - match probe.status { - ToolProbeStatus::Available => Ok(()), - ToolProbeStatus::Missing => anyhow::bail!( - "PDF rendering failed: `tectonic` is not installed.\n\n\ + let registry = ToolRegistry::builtins(); + let inventory = registry.assess_ids_current(["tool.tectonic"]); + if !inventory + .get("tool.tectonic") + .is_some_and(|tool| tool.is_available()) + { + anyhow::bail!( + "PDF rendering failed: `tectonic` is not installed or is incompatible.\n\n\ Fix:\n\ - Install tectonic: https://tectonic-typesetting.github.io/en-US/\n\ - Or configure a different PDF engine" - ), - ToolProbeStatus::TimedOut => anyhow::bail!( - "PDF rendering failed: `tectonic --version` timed out. \ - Verify the tectonic installation before retrying." - ), - ToolProbeStatus::Failed => anyhow::bail!( - "PDF rendering failed: tectonic is installed but its version probe failed: {}", - probe - .diagnostic - .unwrap_or_else(|| "unknown process failure".to_string()) - ), + ); } + Ok(()) } } @@ -122,7 +116,10 @@ mod tests { } fn tectonic_available() -> bool { - ProcessExecutor::new().probe_version("tectonic").is_available() + ToolRegistry::builtins() + .assess_ids_current(["tool.tectonic"]) + .get("tool.tectonic") + .is_some_and(|tool| tool.is_available()) } #[test] diff --git a/crates/renderflow-core/src/toolchain.rs b/crates/renderflow-core/src/toolchain.rs new file mode 100644 index 0000000..563f2e8 --- /dev/null +++ b/crates/renderflow-core/src/toolchain.rs @@ -0,0 +1,1362 @@ +//! Runtime external-tool capability registry and reproducible toolchain evidence. +//! +//! The registry separates tool identity/capability declarations from live host +//! probing. Built-in descriptors are loaded from `data/tool-registry.yaml`, while +//! plugins and YAML command transforms can register additional descriptors at +//! runtime through the same public API. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::env; +use std::fmt; +use std::str::FromStr; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::graph::{Format, MultiTargetDag, TransformGraph}; +use crate::process::{ProcessExecutor, ToolProbeEvidence, ToolProbeStatus as ProcessProbeStatus}; + +pub const TOOL_REGISTRY_SCHEMA: &str = "renderflow.tool-registry/v1"; +pub const TOOLCHAIN_SCHEMA: &str = "renderflow.toolchain/v1"; + +/// Stable machine-readable tool/provider identifier. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ToolId(String); + +impl ToolId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + validate_identifier("tool id", &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for ToolId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl FromStr for ToolId { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + Self::new(value) + } +} + +/// Stable machine-readable capability identifier. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct CapabilityId(String); + +impl CapabilityId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + validate_identifier("capability id", &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for CapabilityId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl FromStr for CapabilityId { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + Self::new(value) + } +} + +fn validate_identifier(kind: &str, value: &str) -> Result<()> { + if value.is_empty() { + anyhow::bail!("{kind} must not be empty"); + } + if !value + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '-')) + { + anyhow::bail!("{kind} '{value}' may contain only ASCII letters, digits, '.', '_' and '-'"); + } + Ok(()) +} + +fn slug(value: &str) -> String { + let mut output = String::new(); + let mut previous_separator = false; + for character in value.chars() { + let normalized = character.to_ascii_lowercase(); + if normalized.is_ascii_alphanumeric() || matches!(normalized, '_' | '-') { + output.push(normalized); + previous_separator = false; + } else if !previous_separator { + output.push('-'); + previous_separator = true; + } + } + output.trim_matches('-').to_string() +} + +/// Expected reproducibility behavior for a provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolDeterminism { + Deterministic, + ConfigurationDependent, + Nondeterministic, +} + +/// Whether a provider runs locally or depends on network/service behavior. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolLocality { + Local, + LocalService, + NetworkOptional, + NetworkRequired, +} + +/// Expected fidelity/loss behavior for the provider as a whole. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolFidelity { + Lossless, + PartialLoss, + Lossy, + PathDependent, +} + +/// Fleet/support importance shown by `doctor`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum ToolSupportTier { + Required, + #[default] + Optional, + Experimental, +} + +impl ToolSupportTier { + pub fn as_str(self) -> &'static str { + match self { + Self::Required => "required", + Self::Optional => "optional", + Self::Experimental => "experimental", + } + } +} + +/// How a tool/provider is discovered at runtime. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ToolDiscovery { + Executable { + candidates: Vec, + #[serde(default = "default_version_args")] + version_args: Vec, + }, + RuntimeService { + service: String, + }, + Virtual, +} + +fn default_version_args() -> Vec { + vec!["--version".to_string()] +} + +/// Minimum/maximum accepted semantic-ish numeric version. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolVersionRequirement { + #[serde(default)] + pub min_inclusive: Option, + #[serde(default)] + pub max_exclusive: Option, +} + +impl ToolVersionRequirement { + pub fn is_unconstrained(&self) -> bool { + self.min_inclusive.is_none() && self.max_exclusive.is_none() + } + + fn supports(&self, version_line: Option<&str>) -> Result { + if self.is_unconstrained() { + return Ok(true); + } + let installed = version_line + .and_then(parse_numeric_version) + .ok_or_else(|| anyhow::anyhow!("version probe did not contain a numeric version"))?; + + if let Some(minimum) = &self.min_inclusive { + let minimum = parse_numeric_version(minimum) + .ok_or_else(|| anyhow::anyhow!("invalid registry minimum version '{minimum}'"))?; + if installed < minimum { + return Ok(false); + } + } + if let Some(maximum) = &self.max_exclusive { + let maximum = parse_numeric_version(maximum) + .ok_or_else(|| anyhow::anyhow!("invalid registry maximum version '{maximum}'"))?; + if installed >= maximum { + return Ok(false); + } + } + Ok(true) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct NumericVersion([u64; 3]); + +fn parse_numeric_version(text: &str) -> Option { + for token in text.split(|character: char| !(character.is_ascii_digit() || character == '.')) { + let token = token.trim_matches('.'); + if token.is_empty() || !token.chars().next().is_some_and(|c| c.is_ascii_digit()) { + continue; + } + let mut parts = [0_u64; 3]; + let mut parsed_any = false; + for (index, part) in token.split('.').take(3).enumerate() { + if part.is_empty() { + break; + } + let Ok(value) = part.parse::() else { + break; + }; + parts[index] = value; + parsed_any = true; + } + if parsed_any { + return Some(NumericVersion(parts)); + } + } + None +} + +fn normalized_numeric_version(text: &str) -> Option { + parse_numeric_version(text) + .map(|version| format!("{}.{}.{}", version.0[0], version.0[1], version.0[2])) +} + +/// Environment requirement without storing or exposing the value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolEnvironmentRequirement { + pub name: String, + #[serde(default)] + pub credential: bool, +} + +/// Canonical provider/tool descriptor. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolDescriptor { + pub id: ToolId, + pub name: String, + pub discovery: ToolDiscovery, + #[serde(default)] + pub version: ToolVersionRequirement, + #[serde(default)] + pub operating_systems: Vec, + #[serde(default)] + pub architectures: Vec, + #[serde(default)] + pub capabilities: Vec, + #[serde(default)] + pub input_media_types: Vec, + #[serde(default)] + pub output_media_types: Vec, + pub determinism: ToolDeterminism, + pub locality: ToolLocality, + pub fidelity: ToolFidelity, + #[serde(default)] + pub required_environment: Vec, + #[serde(default)] + pub required_configuration: Vec, + #[serde(default)] + pub required_services: Vec, + #[serde(default)] + pub fallbacks: Vec, + #[serde(default)] + pub support_tier: ToolSupportTier, + #[serde(default)] + pub license_notes: Option, + #[serde(default)] + pub distribution_notes: Option, +} + +impl ToolDescriptor { + pub fn command(id: ToolId, executable: impl Into) -> Self { + let executable = executable.into(); + Self { + name: executable.clone(), + id, + discovery: ToolDiscovery::Executable { + candidates: vec![executable], + version_args: default_version_args(), + }, + version: ToolVersionRequirement::default(), + operating_systems: Vec::new(), + architectures: Vec::new(), + capabilities: Vec::new(), + input_media_types: Vec::new(), + output_media_types: Vec::new(), + determinism: ToolDeterminism::ConfigurationDependent, + locality: ToolLocality::Local, + fidelity: ToolFidelity::PathDependent, + required_environment: Vec::new(), + required_configuration: Vec::new(), + required_services: Vec::new(), + fallbacks: Vec::new(), + support_tier: ToolSupportTier::Experimental, + license_notes: None, + distribution_notes: Some( + "Dynamically registered command provider; distribution is owned by the host environment" + .to_string(), + ), + } + } + + pub fn virtual_provider(id: ToolId, name: impl Into) -> Self { + Self { + name: name.into(), + id, + discovery: ToolDiscovery::Virtual, + version: ToolVersionRequirement::default(), + operating_systems: Vec::new(), + architectures: Vec::new(), + capabilities: Vec::new(), + input_media_types: Vec::new(), + output_media_types: Vec::new(), + determinism: ToolDeterminism::ConfigurationDependent, + locality: ToolLocality::NetworkOptional, + fidelity: ToolFidelity::PathDependent, + required_environment: Vec::new(), + required_configuration: Vec::new(), + required_services: Vec::new(), + fallbacks: Vec::new(), + support_tier: ToolSupportTier::Experimental, + license_notes: None, + distribution_notes: None, + } + } +} + +#[derive(Debug, Deserialize)] +struct ToolRegistryDocument { + schema: String, + tools: Vec, +} + +/// Runtime facts used to evaluate non-executable requirements. +#[derive(Debug, Clone)] +pub struct ToolRuntimeContext { + pub operating_system: String, + pub architecture: String, + environment_names: BTreeSet, + configuration_keys: BTreeSet, + runtime_services: BTreeSet, +} + +impl ToolRuntimeContext { + pub fn current() -> Self { + Self { + operating_system: env::consts::OS.to_string(), + architecture: env::consts::ARCH.to_string(), + environment_names: env::vars_os() + .map(|(name, _)| name.to_string_lossy().to_ascii_uppercase()) + .collect(), + configuration_keys: BTreeSet::new(), + runtime_services: BTreeSet::new(), + } + } + + pub fn for_platform(os: impl Into, architecture: impl Into) -> Self { + Self { + operating_system: os.into(), + architecture: architecture.into(), + environment_names: BTreeSet::new(), + configuration_keys: BTreeSet::new(), + runtime_services: BTreeSet::new(), + } + } + + pub fn with_environment(mut self, name: impl Into) -> Self { + self.environment_names + .insert(name.into().to_ascii_uppercase()); + self + } + + pub fn with_configuration(mut self, key: impl Into) -> Self { + self.configuration_keys.insert(key.into()); + self + } + + pub fn with_runtime_service(mut self, service: impl Into) -> Self { + self.runtime_services.insert(service.into()); + self + } +} + +/// Probe seam used by tests and embedders to avoid arbitrary host-state dependencies. +pub trait ToolProbe: Send + Sync { + fn probe(&self, executable: &str, version_args: &[String]) -> ToolProbeEvidence; +} + +#[derive(Clone, Default)] +pub struct ProcessToolProbe { + executor: ProcessExecutor, +} + +impl ProcessToolProbe { + pub fn new() -> Self { + Self::default() + } +} + +impl ToolProbe for ProcessToolProbe { + fn probe(&self, executable: &str, version_args: &[String]) -> ToolProbeEvidence { + self.executor + .probe_version_with_args(executable, version_args) + } +} + +/// Canonical availability state used by planner, doctor, SDK, and CLI output. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolAvailabilityStatus { + Available, + MissingExecutable, + UnsupportedVersion, + MissingRuntimeService, + MissingCredential, + MissingConfiguration, + UnsupportedPlatform, + ProbeFailed, + UnknownProvider, +} + +impl ToolAvailabilityStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::MissingExecutable => "missing_executable", + Self::UnsupportedVersion => "unsupported_version", + Self::MissingRuntimeService => "missing_runtime_service", + Self::MissingCredential => "missing_credential", + Self::MissingConfiguration => "missing_configuration", + Self::UnsupportedPlatform => "unsupported_platform", + Self::ProbeFailed => "probe_failed", + Self::UnknownProvider => "unknown_provider", + } + } +} + +/// One evaluated tool/provider. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolAvailability { + pub id: ToolId, + pub name: String, + pub status: ToolAvailabilityStatus, + pub support_tier: ToolSupportTier, + pub selected_executable: Option, + pub version_line: Option, + pub normalized_version: Option, + pub diagnostic: Option, +} + +impl ToolAvailability { + pub fn is_available(&self) -> bool { + self.status == ToolAvailabilityStatus::Available + } + + pub fn summary(&self) -> String { + let detail = self + .diagnostic + .as_deref() + .or(self.version_line.as_deref()) + .unwrap_or("no additional detail"); + format!("{}: {} ({detail})", self.id, self.status.as_str()) + } +} + +/// Deterministically sorted set of evaluated tools. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolInventory { + pub tools: Vec, +} + +impl ToolInventory { + pub fn get(&self, id: &str) -> Option<&ToolAvailability> { + self.tools.iter().find(|tool| tool.id.as_str() == id) + } + + pub fn available_ids(&self) -> HashSet { + self.tools + .iter() + .filter(|tool| tool.is_available()) + .map(|tool| tool.id.to_string()) + .collect() + } + + pub fn blocked_summaries(&self) -> Vec { + self.tools + .iter() + .filter(|tool| !tool.is_available()) + .map(ToolAvailability::summary) + .collect() + } +} + +/// Evidence for a selected provider included in a toolchain fingerprint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SelectedToolEvidence { + pub id: ToolId, + pub executable: Option, + pub version: Option, + pub capabilities: Vec, + pub determinism: ToolDeterminism, + pub locality: ToolLocality, +} + +/// Reproducibility evidence derived only from providers selected by a plan/run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolchainSnapshot { + pub schema: String, + pub fingerprint: String, + pub operating_system: String, + pub architecture: String, + pub selected_tools: Vec, +} + +#[derive(Serialize)] +struct FingerprintMaterial<'a> { + schema: &'a str, + operating_system: &'a str, + architecture: &'a str, + selected_tools: &'a [SelectedToolEvidence], +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolResolutionDecision { + pub id: ToolId, + pub status: ToolAvailabilityStatus, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolResolution { + pub requested: ToolId, + pub selected: Option, + pub decisions: Vec, +} + +/// Public registry that can be extended by plugins without exposing planner internals. +#[derive(Debug, Clone)] +pub struct ToolRegistry { + schema: String, + tools: BTreeMap, +} + +impl ToolRegistry { + pub fn new() -> Self { + Self { + schema: TOOL_REGISTRY_SCHEMA.to_string(), + tools: BTreeMap::new(), + } + } + + pub fn builtins() -> Self { + Self::from_yaml(include_str!("../data/tool-registry.yaml")) + .expect("embedded Renderflow tool registry must be valid") + } + + pub fn from_yaml(yaml: &str) -> Result { + let document: ToolRegistryDocument = + serde_yaml_ng::from_str(yaml).context("failed to parse tool registry YAML")?; + if document.schema != TOOL_REGISTRY_SCHEMA { + anyhow::bail!( + "unsupported tool registry schema '{}'; expected '{}'", + document.schema, + TOOL_REGISTRY_SCHEMA + ); + } + let mut registry = Self { + schema: document.schema, + tools: BTreeMap::new(), + }; + for descriptor in document.tools { + registry.register(descriptor)?; + } + Ok(registry) + } + + pub fn schema(&self) -> &str { + &self.schema + } + + pub fn register(&mut self, mut descriptor: ToolDescriptor) -> Result<&mut Self> { + validate_identifier("tool id", descriptor.id.as_str())?; + for capability in &descriptor.capabilities { + validate_identifier("capability id", capability.as_str())?; + } + descriptor.capabilities.sort(); + descriptor.capabilities.dedup(); + descriptor.fallbacks.sort(); + descriptor.fallbacks.dedup(); + self.tools.insert(descriptor.id.clone(), descriptor); + Ok(self) + } + + pub fn contains(&self, id: &ToolId) -> bool { + self.tools.contains_key(id) + } + + pub fn get(&self, id: &str) -> Option<&ToolDescriptor> { + self.tools.values().find(|tool| tool.id.as_str() == id) + } + + pub fn all(&self) -> impl Iterator { + self.tools.values() + } + + pub fn capabilities(&self) -> BTreeMap> { + let mut capabilities: BTreeMap> = BTreeMap::new(); + for descriptor in self.tools.values() { + for capability in &descriptor.capabilities { + capabilities + .entry(capability.to_string()) + .or_default() + .push(descriptor.id.to_string()); + } + } + for providers in capabilities.values_mut() { + providers.sort(); + providers.dedup(); + } + capabilities + } + + pub fn canonical_id_for_executable(&self, executable: &str) -> ToolId { + for descriptor in self.tools.values() { + if let ToolDiscovery::Executable { candidates, .. } = &descriptor.discovery { + if candidates.iter().any(|candidate| candidate == executable) { + return descriptor.id.clone(); + } + } + } + canonical_tool_id_for_hint(executable) + } + + pub fn ensure_command_provider( + &mut self, + id: ToolId, + executable: impl Into, + capability: CapabilityId, + ) -> Result<&mut Self> { + if !self.tools.contains_key(&id) { + self.register(ToolDescriptor::command(id.clone(), executable))?; + } + self.add_capability(&id, capability)?; + Ok(self) + } + + pub fn ensure_virtual_provider( + &mut self, + id: ToolId, + name: impl Into, + capability: CapabilityId, + ) -> Result<&mut Self> { + if !self.tools.contains_key(&id) { + self.register(ToolDescriptor::virtual_provider(id.clone(), name))?; + } + self.add_capability(&id, capability)?; + Ok(self) + } + + pub fn add_capability(&mut self, id: &ToolId, capability: CapabilityId) -> Result<&mut Self> { + let descriptor = self + .tools + .get_mut(id) + .ok_or_else(|| anyhow::anyhow!("tool provider '{}' is not registered", id))?; + if !descriptor.capabilities.contains(&capability) { + descriptor.capabilities.push(capability); + descriptor.capabilities.sort(); + } + Ok(self) + } + + pub fn assess_all_with( + &self, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, + ) -> ToolInventory { + let ids: Vec = self.tools.keys().map(ToString::to_string).collect(); + self.assess_ids_with(ids, probe, context) + } + + pub fn assess_all_current(&self) -> ToolInventory { + self.assess_all_with(&ProcessToolProbe::new(), &ToolRuntimeContext::current()) + } + + pub fn assess_ids_with( + &self, + ids: I, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, + ) -> ToolInventory + where + I: IntoIterator, + S: AsRef, + { + let unique: BTreeSet = ids.into_iter().map(|id| id.as_ref().to_string()).collect(); + let mut tools = Vec::with_capacity(unique.len()); + for id in unique { + if let Some(descriptor) = self.get(&id) { + tools.push(evaluate_descriptor(descriptor, probe, context)); + } else { + let tool_id = ToolId::new(id.clone()) + .unwrap_or_else(|_| ToolId(format!("tool.unknown.{}", slug(&id)))); + tools.push(ToolAvailability { + id: tool_id, + name: id.clone(), + status: ToolAvailabilityStatus::UnknownProvider, + support_tier: ToolSupportTier::Experimental, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: Some(format!("provider '{id}' is not registered")), + }); + } + } + ToolInventory { tools } + } + + pub fn assess_ids_current(&self, ids: I) -> ToolInventory + where + I: IntoIterator, + S: AsRef, + { + self.assess_ids_with( + ids, + &ProcessToolProbe::new(), + &ToolRuntimeContext::current(), + ) + } + + pub fn fingerprint_selected( + &self, + inventory: &ToolInventory, + ids: I, + context: &ToolRuntimeContext, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + let ids: BTreeSet = ids.into_iter().map(|id| id.as_ref().to_string()).collect(); + let mut selected_tools = Vec::with_capacity(ids.len()); + for id in ids { + let availability = inventory + .get(&id) + .ok_or_else(|| anyhow::anyhow!("tool '{}' was not assessed", id))?; + if !availability.is_available() { + anyhow::bail!( + "cannot fingerprint unavailable provider '{}': {}", + id, + availability.summary() + ); + } + let descriptor = self + .get(&id) + .ok_or_else(|| anyhow::anyhow!("tool '{}' is not registered", id))?; + selected_tools.push(SelectedToolEvidence { + id: descriptor.id.clone(), + executable: availability.selected_executable.clone(), + version: availability + .normalized_version + .clone() + .or_else(|| availability.version_line.clone()), + capabilities: descriptor.capabilities.clone(), + determinism: descriptor.determinism, + locality: descriptor.locality, + }); + } + selected_tools.sort_by(|left, right| left.id.cmp(&right.id)); + + let material = FingerprintMaterial { + schema: TOOLCHAIN_SCHEMA, + operating_system: &context.operating_system, + architecture: &context.architecture, + selected_tools: &selected_tools, + }; + let encoded = serde_json::to_vec(&material) + .context("failed to serialize toolchain fingerprint material")?; + let digest = Sha256::digest(encoded); + Ok(ToolchainSnapshot { + schema: TOOLCHAIN_SCHEMA.to_string(), + fingerprint: format!("sha256:{digest:x}"), + operating_system: context.operating_system.clone(), + architecture: context.architecture.clone(), + selected_tools, + }) + } + + pub fn fingerprint_for_dag( + &self, + inventory: &ToolInventory, + dag: &MultiTargetDag, + context: &ToolRuntimeContext, + ) -> Result { + self.fingerprint_selected( + inventory, + dag.all_edges() + .iter() + .filter_map(|edge| edge.provider_id.as_deref()), + context, + ) + } + + pub fn resolve_with( + &self, + requested: &ToolId, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, + ) -> ToolResolution { + let mut queue = vec![requested.clone()]; + let mut visited = BTreeSet::new(); + let mut decisions = Vec::new(); + + while let Some(id) = queue.first().cloned() { + queue.remove(0); + if !visited.insert(id.clone()) { + continue; + } + let Some(descriptor) = self.tools.get(&id) else { + decisions.push(ToolResolutionDecision { + id, + status: ToolAvailabilityStatus::UnknownProvider, + reason: "provider is not registered".to_string(), + }); + continue; + }; + let availability = evaluate_descriptor(descriptor, probe, context); + decisions.push(ToolResolutionDecision { + id: id.clone(), + status: availability.status, + reason: availability.summary(), + }); + if availability.is_available() { + return ToolResolution { + requested: requested.clone(), + selected: Some(id), + decisions, + }; + } + queue.extend(descriptor.fallbacks.iter().cloned()); + } + + ToolResolution { + requested: requested.clone(), + selected: None, + decisions, + } + } +} + +impl Default for ToolRegistry { + fn default() -> Self { + Self::new() + } +} + +fn evaluate_descriptor( + descriptor: &ToolDescriptor, + probe: &dyn ToolProbe, + context: &ToolRuntimeContext, +) -> ToolAvailability { + let unavailable = |status: ToolAvailabilityStatus, diagnostic: String| ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status, + support_tier: descriptor.support_tier, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: Some(diagnostic), + }; + + if !descriptor.operating_systems.is_empty() + && !descriptor + .operating_systems + .iter() + .any(|os| os == &context.operating_system) + { + return unavailable( + ToolAvailabilityStatus::UnsupportedPlatform, + format!( + "platform '{}' is unsupported; expected one of {}", + context.operating_system, + descriptor.operating_systems.join(", ") + ), + ); + } + if !descriptor.architectures.is_empty() + && !descriptor + .architectures + .iter() + .any(|architecture| architecture == &context.architecture) + { + return unavailable( + ToolAvailabilityStatus::UnsupportedPlatform, + format!( + "architecture '{}' is unsupported; expected one of {}", + context.architecture, + descriptor.architectures.join(", ") + ), + ); + } + + for requirement in &descriptor.required_environment { + if !context + .environment_names + .contains(&requirement.name.to_ascii_uppercase()) + { + return unavailable( + if requirement.credential { + ToolAvailabilityStatus::MissingCredential + } else { + ToolAvailabilityStatus::MissingConfiguration + }, + format!( + "required environment variable '{}' is not set", + requirement.name + ), + ); + } + } + for key in &descriptor.required_configuration { + if !context.configuration_keys.contains(key) { + return unavailable( + ToolAvailabilityStatus::MissingConfiguration, + format!("required configuration key '{key}' is not available"), + ); + } + } + for service in &descriptor.required_services { + if !context.runtime_services.contains(service) { + return unavailable( + ToolAvailabilityStatus::MissingRuntimeService, + format!("required runtime service '{service}' is unavailable"), + ); + } + } + + match &descriptor.discovery { + ToolDiscovery::Virtual => ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::Available, + support_tier: descriptor.support_tier, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: None, + }, + ToolDiscovery::RuntimeService { service } => { + if context.runtime_services.contains(service) { + ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::Available, + support_tier: descriptor.support_tier, + selected_executable: None, + version_line: None, + normalized_version: None, + diagnostic: Some(format!("runtime service '{service}' is available")), + } + } else { + unavailable( + ToolAvailabilityStatus::MissingRuntimeService, + format!("runtime service '{service}' is unavailable"), + ) + } + } + ToolDiscovery::Executable { + candidates, + version_args, + } => { + let mut saw_probe_failure = None; + for executable in candidates { + let evidence = probe.probe(executable, version_args); + match evidence.status { + ProcessProbeStatus::Available => { + let version_line = evidence.version_line.clone(); + match descriptor.version.supports(version_line.as_deref()) { + Ok(true) => { + return ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::Available, + support_tier: descriptor.support_tier, + selected_executable: Some(executable.clone()), + normalized_version: version_line + .as_deref() + .and_then(normalized_numeric_version), + version_line, + diagnostic: None, + }; + } + Ok(false) => { + return ToolAvailability { + id: descriptor.id.clone(), + name: descriptor.name.clone(), + status: ToolAvailabilityStatus::UnsupportedVersion, + support_tier: descriptor.support_tier, + selected_executable: Some(executable.clone()), + normalized_version: version_line + .as_deref() + .and_then(normalized_numeric_version), + diagnostic: Some(format!( + "installed version does not satisfy registry requirement {:?}", + descriptor.version + )), + version_line, + }; + } + Err(error) => { + return unavailable( + ToolAvailabilityStatus::UnsupportedVersion, + error.to_string(), + ); + } + } + } + ProcessProbeStatus::Missing => {} + ProcessProbeStatus::TimedOut | ProcessProbeStatus::Failed => { + saw_probe_failure = Some( + evidence + .diagnostic + .unwrap_or_else(|| "version probe failed".to_string()), + ); + } + } + } + if let Some(diagnostic) = saw_probe_failure { + unavailable(ToolAvailabilityStatus::ProbeFailed, diagnostic) + } else { + unavailable( + ToolAvailabilityStatus::MissingExecutable, + format!("none of [{}] were found", candidates.join(", ")), + ) + } + } + } +} + +/// Map known executable/tool hints to stable built-in IDs; arbitrary hints get +/// a stable dynamic-command namespace. +pub fn canonical_tool_id_for_hint(hint: &str) -> ToolId { + let canonical = match hint { + "pandoc" => "tool.pandoc".to_string(), + "tectonic" => "tool.tectonic".to_string(), + "ffmpeg" => "tool.ffmpeg".to_string(), + "wkhtmltopdf" => "tool.wkhtmltopdf".to_string(), + "zip" => "tool.zip".to_string(), + "img2pdf" => "tool.img2pdf".to_string(), + "gs" | "ghostscript" => "tool.ghostscript".to_string(), + value => format!("tool.command.{}", slug(value)), + }; + ToolId::new(canonical).expect("canonicalized tool id is valid") +} + +/// Stable capability ID for a concrete format transformation. +pub fn transform_capability_id(from: Format, to: Format) -> CapabilityId { + CapabilityId::new(format!("transform.{}.{}", from, to)) + .expect("Format display values produce valid capability ids") +} + +/// Assess all provider IDs referenced by a graph and return an availability-filtered graph. +pub fn filter_graph_for_current_toolchain( + graph: &TransformGraph, + registry: &ToolRegistry, +) -> (TransformGraph, ToolInventory, ToolRuntimeContext) { + let context = ToolRuntimeContext::current(); + let inventory = + registry.assess_ids_with(graph.provider_ids(), &ProcessToolProbe::new(), &context); + let available = inventory.available_ids(); + let filtered = graph.filtered_by_available_providers(&available); + (filtered, inventory, context) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::process::{ProcessPlatform, ProcessTreeTermination}; + + #[derive(Default)] + struct FakeProbe { + responses: BTreeMap, + } + + impl FakeProbe { + fn with( + mut self, + executable: &str, + status: ProcessProbeStatus, + version: Option<&str>, + ) -> Self { + self.responses.insert( + executable.to_string(), + ToolProbeEvidence { + executable: executable.to_string(), + status, + version_line: version.map(str::to_string), + duration_ms: 1, + platform: ProcessPlatform { + os: "test", + arch: "test", + tree_termination: ProcessTreeTermination::DirectChild, + }, + diagnostic: None, + }, + ); + self + } + } + + impl ToolProbe for FakeProbe { + fn probe(&self, executable: &str, _version_args: &[String]) -> ToolProbeEvidence { + self.responses + .get(executable) + .cloned() + .unwrap_or(ToolProbeEvidence { + executable: executable.to_string(), + status: ProcessProbeStatus::Missing, + version_line: None, + duration_ms: 0, + platform: ProcessPlatform { + os: "test", + arch: "test", + tree_termination: ProcessTreeTermination::DirectChild, + }, + diagnostic: Some("missing".to_string()), + }) + } + } + + fn test_descriptor(id: &str, executable: &str) -> ToolDescriptor { + ToolDescriptor { + id: ToolId::new(id).unwrap(), + name: id.to_string(), + discovery: ToolDiscovery::Executable { + candidates: vec![executable.to_string()], + version_args: vec!["--version".to_string()], + }, + version: ToolVersionRequirement::default(), + operating_systems: Vec::new(), + architectures: Vec::new(), + capabilities: vec![CapabilityId::new("transform.test").unwrap()], + input_media_types: Vec::new(), + output_media_types: Vec::new(), + determinism: ToolDeterminism::Deterministic, + locality: ToolLocality::Local, + fidelity: ToolFidelity::Lossless, + required_environment: Vec::new(), + required_configuration: Vec::new(), + required_services: Vec::new(), + fallbacks: Vec::new(), + support_tier: ToolSupportTier::Optional, + license_notes: None, + distribution_notes: None, + } + } + + #[test] + fn embedded_registry_contains_core_wrapped_tools() { + let registry = ToolRegistry::builtins(); + assert!(registry.get("tool.pandoc").is_some()); + assert!(registry.get("tool.tectonic").is_some()); + assert!(registry.get("tool.ffmpeg").is_some()); + assert!(registry.get("tool.ghostscript").is_some()); + } + + #[test] + fn stable_ids_reject_whitespace() { + assert!(ToolId::new("tool.bad id").is_err()); + assert!(CapabilityId::new("bad capability").is_err()); + } + + #[test] + fn missing_executable_is_distinct() { + let mut registry = ToolRegistry::new(); + registry + .register(test_descriptor("tool.fake", "fake")) + .unwrap(); + let inventory = registry.assess_all_with( + &FakeProbe::default(), + &ToolRuntimeContext::for_platform("linux", "x86_64"), + ); + assert_eq!( + inventory.tools[0].status, + ToolAvailabilityStatus::MissingExecutable + ); + } + + #[test] + fn unsupported_version_is_distinct() { + let mut descriptor = test_descriptor("tool.fake", "fake"); + descriptor.version.min_inclusive = Some("2.0.0".to_string()); + let mut registry = ToolRegistry::new(); + registry.register(descriptor).unwrap(); + let probe = + FakeProbe::default().with("fake", ProcessProbeStatus::Available, Some("fake 1.9.0")); + let inventory = + registry.assess_all_with(&probe, &ToolRuntimeContext::for_platform("linux", "x86_64")); + assert_eq!( + inventory.tools[0].status, + ToolAvailabilityStatus::UnsupportedVersion + ); + } + + #[test] + fn doctor_states_distinguish_service_credential_config_and_platform() { + let mut registry = ToolRegistry::new(); + + let mut service = + ToolDescriptor::virtual_provider(ToolId::new("tool.service").unwrap(), "service"); + service.discovery = ToolDiscovery::RuntimeService { + service: "daemon".to_string(), + }; + registry.register(service).unwrap(); + + let mut credential = + ToolDescriptor::virtual_provider(ToolId::new("tool.credential").unwrap(), "credential"); + credential + .required_environment + .push(ToolEnvironmentRequirement { + name: "FAKE_TOKEN".to_string(), + credential: true, + }); + registry.register(credential).unwrap(); + + let mut configuration = ToolDescriptor::virtual_provider( + ToolId::new("tool.configuration").unwrap(), + "configuration", + ); + configuration + .required_configuration + .push("profile".to_string()); + registry.register(configuration).unwrap(); + + let mut platform = + ToolDescriptor::virtual_provider(ToolId::new("tool.platform").unwrap(), "platform"); + platform.operating_systems = vec!["plan9".to_string()]; + registry.register(platform).unwrap(); + + let inventory = registry.assess_all_with( + &FakeProbe::default(), + &ToolRuntimeContext::for_platform("linux", "x86_64"), + ); + assert_eq!( + inventory.get("tool.service").unwrap().status, + ToolAvailabilityStatus::MissingRuntimeService + ); + assert_eq!( + inventory.get("tool.credential").unwrap().status, + ToolAvailabilityStatus::MissingCredential + ); + assert_eq!( + inventory.get("tool.configuration").unwrap().status, + ToolAvailabilityStatus::MissingConfiguration + ); + assert_eq!( + inventory.get("tool.platform").unwrap().status, + ToolAvailabilityStatus::UnsupportedPlatform + ); + } + + #[test] + fn fallback_resolution_explains_why_substitute_won() { + let mut primary = test_descriptor("tool.primary", "primary"); + primary + .fallbacks + .push(ToolId::new("tool.fallback").unwrap()); + let fallback = test_descriptor("tool.fallback", "fallback"); + let mut registry = ToolRegistry::new(); + registry.register(primary).unwrap(); + registry.register(fallback).unwrap(); + let probe = FakeProbe::default().with( + "fallback", + ProcessProbeStatus::Available, + Some("fallback 3.0.0"), + ); + let resolution = registry.resolve_with( + &ToolId::new("tool.primary").unwrap(), + &probe, + &ToolRuntimeContext::for_platform("linux", "x86_64"), + ); + assert_eq!( + resolution.selected.as_ref().map(ToolId::as_str), + Some("tool.fallback") + ); + assert_eq!(resolution.decisions.len(), 2); + assert_eq!( + resolution.decisions[0].status, + ToolAvailabilityStatus::MissingExecutable + ); + } + + #[test] + fn fingerprint_is_order_independent_and_selected_only() { + let mut registry = ToolRegistry::new(); + registry.register(test_descriptor("tool.a", "a")).unwrap(); + registry.register(test_descriptor("tool.b", "b")).unwrap(); + let probe = FakeProbe::default() + .with("a", ProcessProbeStatus::Available, Some("a 1.2.3")) + .with("b", ProcessProbeStatus::Available, Some("b 4.5.6")); + let context = ToolRuntimeContext::for_platform("linux", "x86_64"); + let inventory = registry.assess_all_with(&probe, &context); + let first = registry + .fingerprint_selected(&inventory, ["tool.b", "tool.a"], &context) + .unwrap(); + let second = registry + .fingerprint_selected(&inventory, ["tool.a", "tool.b"], &context) + .unwrap(); + let only_a = registry + .fingerprint_selected(&inventory, ["tool.a"], &context) + .unwrap(); + assert_eq!(first.fingerprint, second.fingerprint); + assert_ne!(first.fingerprint, only_a.fingerprint); + } + + #[test] + fn dynamic_command_provider_can_be_augmented_with_transform_capability() { + let mut registry = ToolRegistry::builtins(); + let id = registry.canonical_id_for_executable("made-up-renderer"); + registry + .ensure_command_provider( + id.clone(), + "made-up-renderer", + CapabilityId::new("transform.markdown.pdf").unwrap(), + ) + .unwrap(); + let descriptor = registry.get(id.as_str()).unwrap(); + assert!(descriptor + .capabilities + .iter() + .any(|capability| capability.as_str() == "transform.markdown.pdf")); + } +} diff --git a/crates/renderflow-core/src/transforms/aggregation.rs b/crates/renderflow-core/src/transforms/aggregation.rs index cc6096e..4651681 100644 --- a/crates/renderflow-core/src/transforms/aggregation.rs +++ b/crates/renderflow-core/src/transforms/aggregation.rs @@ -8,7 +8,7 @@ use anyhow::{Context, Result}; use tracing::{debug, info}; use crate::process::{ - is_explicit_shell_invocation, ProcessExpectedOutput, ProcessExecutor, ProcessInput, + is_explicit_shell_invocation, ProcessExecutor, ProcessExpectedOutput, ProcessInput, ProcessOutputMode, ProcessRequest, DEFAULT_CAPTURE_LIMIT_BYTES, DEFAULT_PROCESS_TIMEOUT, }; @@ -42,7 +42,9 @@ impl AggregationRegistry { } pub fn get(&self, name: &str) -> Option<&dyn AggregationTransform> { - self.transforms.get(name).map(|transform| transform.as_ref()) + self.transforms + .get(name) + .map(|transform| transform.as_ref()) } pub fn apply(&self, name: &str, inputs: &[&str], output_path: &str) -> Result<()> { @@ -266,7 +268,10 @@ mod tests { registry .apply("join", &["page1", "page2", "page3"], out.to_str().unwrap()) .unwrap(); - assert_eq!(std::fs::read_to_string(&out).unwrap(), "page1\npage2\npage3"); + assert_eq!( + std::fs::read_to_string(&out).unwrap(), + "page1\npage2\npage3" + ); } #[test] @@ -297,9 +302,7 @@ mod tests { ); let dir = tempfile::tempdir().unwrap(); let out = dir.path().join("out.txt"); - assert!(transform - .aggregate(&["a"], out.to_str().unwrap()) - .is_err()); + assert!(transform.aggregate(&["a"], out.to_str().unwrap()).is_err()); } #[cfg(unix)] diff --git a/crates/renderflow-core/src/transforms/command.rs b/crates/renderflow-core/src/transforms/command.rs index 2c68f51..d6ab6f5 100644 --- a/crates/renderflow-core/src/transforms/command.rs +++ b/crates/renderflow-core/src/transforms/command.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use super::Transform; use crate::process::{ - is_explicit_shell_invocation, ProcessExpectedOutput, ProcessExecutor, ProcessInput, + is_explicit_shell_invocation, ProcessExecutor, ProcessExpectedOutput, ProcessInput, ProcessOutputMode, ProcessRequest, DEFAULT_CAPTURE_LIMIT_BYTES, DEFAULT_PROCESS_TIMEOUT, }; diff --git a/crates/renderflow-core/src/transforms/yaml_loader.rs b/crates/renderflow-core/src/transforms/yaml_loader.rs index 1d02def..7938efd 100644 --- a/crates/renderflow-core/src/transforms/yaml_loader.rs +++ b/crates/renderflow-core/src/transforms/yaml_loader.rs @@ -12,6 +12,7 @@ use super::{ Transform, TransformRegistry, }; use crate::ai::providers::{OllamaProvider, OpenAiProvider}; +use crate::toolchain::{transform_capability_id, ToolId, ToolRegistry}; /// Top-level structure of a YAML transform configuration file. /// @@ -62,6 +63,9 @@ pub struct YamlTransformDef { /// those fields is set. #[serde(default)] pub program: Option, + /// Optional stable provider ID. When omitted, Renderflow infers one from program/plugin/AI. + #[serde(default)] + pub provider: Option, /// Arguments passed to the program. /// /// Use `{input}` as a placeholder for a temporary file that contains the @@ -176,6 +180,11 @@ impl YamlTransformDef { if self.name.trim().is_empty() { anyhow::bail!("transform 'name' must not be empty"); } + if let Some(provider) = &self.provider { + ToolId::new(provider.clone()).with_context(|| { + format!("transform '{}': invalid stable provider id", self.name) + })?; + } match (&self.ai, &self.plugin, &self.program) { // ai takes precedence – validate the backend name. (Some(backend), _, _) if backend.trim().is_empty() => { @@ -364,6 +373,36 @@ impl YamlTransformDef { .unwrap_or(false) } + /// Resolve the stable provider ID used by planning/toolchain evidence. + pub fn provider_id(&self, registry: &ToolRegistry) -> Result { + if let Some(provider) = &self.provider { + return ToolId::new(provider.clone()); + } + if let Some(program) = &self.program { + return Ok(registry.canonical_id_for_executable(program)); + } + if let Some(plugin) = &self.plugin { + let component = plugin + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { + character.to_ascii_lowercase() + } else { + '-' + } + }) + .collect::(); + return ToolId::new(format!("tool.plugin.{component}")); + } + if let Some(ai) = &self.ai { + return ToolId::new(format!("tool.ai.{}", ai.to_ascii_lowercase())); + } + anyhow::bail!( + "transform '{}': cannot resolve provider identity", + self.name + ) + } + /// Build a [`CommandAggregationTransform`] from this definition. /// /// Only valid when `program` is set; call [`validate`](Self::validate) @@ -540,34 +579,75 @@ pub fn load_aggregation_transforms_from_yaml(path: &str) -> Result Result { + let content = fs::read_to_string(path) + .with_context(|| format!("Failed to read transform config: {}", path))?; + parse_tool_registry_from_str(&content) + .with_context(|| format!("Failed to load tool registry from: {}", path)) +} + +/// Build the runtime provider registry implied by a transform YAML document. +pub fn parse_tool_registry_from_str(yaml: &str) -> Result { + let config: YamlTransformConfig = + serde_yaml_ng::from_str(yaml).context("Failed to parse YAML transform config")?; + let mut registry = ToolRegistry::builtins(); + + for def in &config.transforms { + def.validate()?; + let from: crate::graph::Format = def.from.parse()?; + let to: crate::graph::Format = def.to.parse()?; + let capability = transform_capability_id(from, to); + let provider = def.provider_id(®istry)?; + + if let Some(program) = &def.program { + registry.ensure_command_provider(provider, program.clone(), capability)?; + } else { + let provider_name = def + .plugin + .as_deref() + .or(def.ai.as_deref()) + .unwrap_or(def.name.as_str()); + registry.ensure_virtual_provider(provider, provider_name, capability)?; + } + } + + Ok(registry) +} + +/// Build graph, executor, and canonical runtime tool registry from a YAML file. +pub fn build_graph_executor_and_tools_from_yaml( path: &str, -) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { +) -> Result<( + crate::graph::TransformGraph, + crate::graph::DagExecutor, + ToolRegistry, +)> { let content = fs::read_to_string(path) .with_context(|| format!("Failed to read transform config: {}", path))?; - build_graph_and_executor_from_str(&content) - .with_context(|| format!("Failed to build graph and executor from: {}", path)) + build_graph_executor_and_tools_from_str(&content) + .with_context(|| format!("Failed to build graph/executor/tools from: {}", path)) } -/// Build a [`TransformGraph`] and a [`DagExecutor`] from a YAML string. -/// -/// See [`build_graph_and_executor_from_yaml`] for details. -pub fn build_graph_and_executor_from_str( +/// Build graph, executor, and canonical runtime tool registry from YAML text. +pub fn build_graph_executor_and_tools_from_str( yaml: &str, -) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { +) -> Result<( + crate::graph::TransformGraph, + crate::graph::DagExecutor, + ToolRegistry, +)> { use std::sync::Arc; use crate::graph::{DagExecutor, Format, InputKind, TransformEdge, TransformGraph}; let config: YamlTransformConfig = serde_yaml_ng::from_str(yaml).context("Failed to parse YAML transform config")?; - + let tool_registry = parse_tool_registry_from_str(yaml)?; let mut graph = TransformGraph::new(); let mut executor = DagExecutor::new(); for def in &config.transforms { def.validate()?; - let from: Format = def .from .parse() @@ -576,24 +656,27 @@ pub fn build_graph_and_executor_from_str( .to .parse() .with_context(|| format!("transform '{}': invalid 'to' format", def.name))?; - let input_kind = if def.is_collection() { InputKind::Collection } else { InputKind::Single }; + let provider = def.provider_id(&tool_registry)?; + let capability = transform_capability_id(from, to); - graph.add_transform(TransformEdge::with_input_kind( - from, - to, - def.cost, - def.quality, - input_kind, - )); + graph.add_transform( + TransformEdge::with_input_kind(from, to, def.cost, def.quality, input_kind) + .with_provider(provider.to_string(), capability.to_string()), + ); if def.is_collection() { let agg = Arc::new(def.to_aggregation_transform()?); executor.register_aggregation(from, to, agg); + } else if def.plugin.is_some() { + anyhow::bail!( + "transform '{}': graph execution requires plugin registration through an embedding registry", + def.name + ); } else { let transform: Arc = if def.ai.is_some() { Arc::new(def.to_ai_transform()?) @@ -604,6 +687,22 @@ pub fn build_graph_and_executor_from_str( } } + Ok((graph, executor, tool_registry)) +} + +/// Compatibility wrapper returning only graph and executor. +pub fn build_graph_and_executor_from_yaml( + path: &str, +) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { + let (graph, executor, _tools) = build_graph_executor_and_tools_from_yaml(path)?; + Ok((graph, executor)) +} + +/// Compatibility wrapper returning only graph and executor. +pub fn build_graph_and_executor_from_str( + yaml: &str, +) -> Result<(crate::graph::TransformGraph, crate::graph::DagExecutor)> { + let (graph, executor, _tools) = build_graph_executor_and_tools_from_str(yaml)?; Ok((graph, executor)) } diff --git a/docs/cli-reference/tools.md b/docs/cli-reference/tools.md new file mode 100644 index 0000000..15ecbbd --- /dev/null +++ b/docs/cli-reference/tools.md @@ -0,0 +1,37 @@ +# tools and capabilities + +Renderflow exposes the runtime provider registry through the same structured model used by planning and diagnostics. + +## List providers + +```bash +renderflow tools list +renderflow tools list --format json +renderflow tools list --format yaml +``` + +Use `--transforms ` to include providers inferred from a transform YAML file, including arbitrary command providers that are not part of the built-in catalog. + +## Inspect one provider + +```bash +renderflow tools inspect tool.ffmpeg +renderflow tools inspect tool.pandoc --format json +``` + +Inspection includes discovery strategy, live availability state, selected executable, installed version evidence, determinism/locality/fidelity metadata, capability IDs, fallbacks, and diagnostics. + +## List capabilities + +```bash +renderflow capabilities +renderflow capabilities --format json +``` + +Capability IDs and provider IDs are stable machine-readable identifiers. Human-readable CLI output is rendered from the same data returned by JSON/YAML modes. + +## Toolchain fingerprints + +Graph planning fingerprints only providers selected by the final DAG. The fingerprint includes the selected provider IDs, compatible installed versions, relevant executable identity, provider capability metadata, and the target OS/architecture. It does **not** hash the entire host environment. + +Graph execution uses that fingerprint in artifact-cache compatibility and writes the selected toolchain evidence into Renderflow state for reproducibility/provenance consumers. diff --git a/docs/user-guide/tool-registry.md b/docs/user-guide/tool-registry.md new file mode 100644 index 0000000..d349219 --- /dev/null +++ b/docs/user-guide/tool-registry.md @@ -0,0 +1,59 @@ +# Tool Registry + +!!! info + This page is generated from the canonical built-in catalog at + `crates/renderflow-core/data/tool-registry.yaml` by + `scripts/generate_tool_registry_doc.py`. Do not edit it by hand. + +Registry schema: `renderflow.tool-registry/v1` + +The built-in catalog defines stable provider IDs, discovery/version probes, +platform constraints, capability IDs, determinism/locality/fidelity metadata, +runtime requirements, fallback relationships, and distribution/licensing notes. +YAML command transforms and plugins can register additional runtime providers +without editing this built-in catalog. + +## Built-in providers + +| Provider ID | Name | Discovery | Tier | Determinism | Locality | +| --- | --- | --- | --- | --- | --- | +| `tool.ffmpeg` | FFmpeg | executable: `ffmpeg` | optional | configuration_dependent | local | +| `tool.ghostscript` | Ghostscript | executable: `gs` | experimental | configuration_dependent | local | +| `tool.img2pdf` | img2pdf | executable: `img2pdf` | experimental | deterministic | local | +| `tool.pandoc` | Pandoc | executable: `pandoc` | required | configuration_dependent | local | +| `tool.tectonic` | Tectonic | executable: `tectonic` | optional | configuration_dependent | network_optional | +| `tool.wkhtmltopdf` | wkhtmltopdf | executable: `wkhtmltopdf` | experimental | configuration_dependent | local | +| `tool.zip` | Info-ZIP compatible zip | executable: `zip` | experimental | configuration_dependent | local | + +## Capability matrix + +| Capability ID | Provider ID | +| --- | --- | +| `audio.convert` | `tool.ffmpeg` | +| `image.convert` | `tool.ffmpeg` | +| `media.convert` | `tool.ffmpeg` | +| `video.convert` | `tool.ffmpeg` | +| `pdf.process` | `tool.ghostscript` | +| `tiff.aggregate.press_pdf` | `tool.ghostscript` | +| `image.aggregate.pdf` | `tool.img2pdf` | +| `document.convert` | `tool.pandoc` | +| `document.generate` | `tool.pandoc` | +| `latex.compile` | `tool.tectonic` | +| `pdf.typeset` | `tool.tectonic` | +| `html.render.pdf` | `tool.wkhtmltopdf` | +| `archive.zip.create` | `tool.zip` | +| `comic.cbz.create` | `tool.zip` | + +## Runtime inspection + +Use the CLI to inspect the live host using the same model: + +```text +renderflow tools list +renderflow tools inspect tool.ffmpeg +renderflow capabilities +renderflow doctor +``` + +Add `--format json` or `--format yaml` to the tools/capabilities commands +when machine-readable evidence is needed. diff --git a/mkdocs.yml b/mkdocs.yml index c545d92..64f0967 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ nav: - User Guide: - Configuration: user-guide/configuration.md - Supported Formats: user-guide/supported-formats.md + - Tool Registry: user-guide/tool-registry.md - Pipelines: user-guide/pipelines.md - Transforms: user-guide/transforms.md - Optimization: user-guide/optimization.md @@ -117,6 +118,7 @@ nav: - graph: cli-reference/graph.md - plugin: cli-reference/plugin.md - ai: cli-reference/ai.md + - tools & capabilities: cli-reference/tools.md - Transform Reference: - Overview: transform-reference/index.md - Emoji: transform-reference/emoji.md diff --git a/scripts/generate_tool_registry_doc.py b/scripts/generate_tool_registry_doc.py new file mode 100755 index 0000000..67643c4 --- /dev/null +++ b/scripts/generate_tool_registry_doc.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = ROOT / "crates" / "renderflow-core" / "data" / "tool-registry.yaml" +DOC_PATH = ROOT / "docs" / "user-guide" / "tool-registry.md" + + +def render_table(headers: list[str], rows: list[list[str]]) -> list[str]: + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + for row in rows: + lines.append("| " + " | ".join(row) + " |") + return lines + + +def discovery_label(discovery: dict[str, object]) -> str: + kind = str(discovery.get("kind", "unknown")) + if kind == "executable": + candidates = discovery.get("candidates", []) + return "executable: " + ", ".join(f"`{value}`" for value in candidates) + if kind == "runtime_service": + return f"service: `{discovery.get('service', '-')}`" + return kind + + +def main() -> None: + registry = yaml.safe_load(REGISTRY_PATH.read_text(encoding="utf-8")) + schema = registry["schema"] + tools = sorted(registry.get("tools", []), key=lambda tool: tool["id"]) + + tool_rows: list[list[str]] = [] + capability_rows: list[list[str]] = [] + for tool in tools: + tool_rows.append( + [ + f"`{tool['id']}`", + str(tool["name"]), + discovery_label(tool["discovery"]), + str(tool.get("support_tier", "optional")), + str(tool.get("determinism", "-")), + str(tool.get("locality", "-")), + ] + ) + for capability in sorted(tool.get("capabilities", [])): + capability_rows.append([f"`{capability}`", f"`{tool['id']}`"]) + + content = [ + "# Tool Registry", + "", + "!!! info", + " This page is generated from the canonical built-in catalog at", + " `crates/renderflow-core/data/tool-registry.yaml` by", + " `scripts/generate_tool_registry_doc.py`. Do not edit it by hand.", + "", + f"Registry schema: `{schema}`", + "", + "The built-in catalog defines stable provider IDs, discovery/version probes,", + "platform constraints, capability IDs, determinism/locality/fidelity metadata,", + "runtime requirements, fallback relationships, and distribution/licensing notes.", + "YAML command transforms and plugins can register additional runtime providers", + "without editing this built-in catalog.", + "", + "## Built-in providers", + "", + *render_table( + ["Provider ID", "Name", "Discovery", "Tier", "Determinism", "Locality"], + tool_rows, + ), + "", + "## Capability matrix", + "", + *render_table(["Capability ID", "Provider ID"], capability_rows), + "", + "## Runtime inspection", + "", + "Use the CLI to inspect the live host using the same model:", + "", + "```text", + "renderflow tools list", + "renderflow tools inspect tool.ffmpeg", + "renderflow capabilities", + "renderflow doctor", + "```", + "", + "Add `--format json` or `--format yaml` to the tools/capabilities commands", + "when machine-readable evidence is needed.", + "", + ] + + DOC_PATH.write_text("\n".join(content), encoding="utf-8") + + +if __name__ == "__main__": + main()