diff --git a/src/lib.rs b/src/lib.rs index 286513b..b2820f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,7 +23,7 @@ //! # use std::collections::HashMap; //! let mut variables = HashMap::new(); //! variables.insert("name", "world"); -//! assert_eq!(subst::substitute("Hello $name!", &variables)?, "Hello world!"); +//! assert_eq!(subst::substitute("Hello $name!", &variables, subst::Mode::Strict)?, "Hello world!"); //! # Ok(()) //! # } //! ``` @@ -34,8 +34,8 @@ //! # fn main() -> Result<(), subst::Error> { //! # std::env::set_var("XDG_CONFIG_HOME", "/home/user/.config"); //! assert_eq!( -//! subst::substitute("$XDG_CONFIG_HOME/my-app/config.toml", &subst::Env)?, -//! "/home/user/.config/my-app/config.toml", +//! subst::substitute("$XDG_CONFIG_HOME/my-app/config.toml", &subst::Env, subst::Mode::Strict)?, +//! "/home/user/.config/my-app/config.toml" //! ); //! # Ok(()) //! # } @@ -48,7 +48,7 @@ //! # use std::collections::HashMap; //! let mut variables = HashMap::new(); //! variables.insert("name", b"world"); -//! assert_eq!(subst::substitute_bytes(b"Hello $name!", &variables)?, b"Hello world!"); +//! assert_eq!(subst::substitute_bytes(b"Hello $name!", &variables, subst::Mode::Strict)?, b"Hello world!"); //! # Ok(()) //! # } //! ``` @@ -63,6 +63,17 @@ pub use map::*; #[cfg(feature = "yaml")] pub mod yaml; +/// Substitution mode +#[derive(Debug, Copy, Clone)] +pub enum Mode { + /// Do not allow missing variables. + Strict, + /// Do not substitute missing variables without default. + Partial, + /// Substitute missing variables with and empty string. + Permissive, +} + /// Substitute variables in a string. /// /// Variables have the form `$NAME`, `${NAME}` or `${NAME:default}`. @@ -73,15 +84,20 @@ pub mod yaml; /// /// You can pass either a [`HashMap`][std::collections::HashMap], [`BTreeMap`][std::collections::BTreeMap] or [`Env`] as the `variables` parameter. /// The maps must have [`&str`] or [`String`] keys, and the values must be [`AsRef`]. -pub fn substitute<'a, M>(source: &str, variables: &'a M) -> Result +pub fn substitute<'a, M>(source: &str, variables: &'a M, mode: Mode) -> Result where M: VariableMap<'a> + ?Sized, M::Value: AsRef, { let mut output = Vec::with_capacity(source.len() + source.len() / 10); - substitute_impl(&mut output, source.as_bytes(), 0..source.len(), variables, &|x| { - x.as_ref().as_bytes() - })?; + substitute_impl( + &mut output, + source.as_bytes(), + 0..source.len(), + variables, + &|x| x.as_ref().as_bytes(), + mode, + )?; // SAFETY: Both source and all variable values are valid UTF-8, so substitation result is also valid UTF-8. unsafe { Ok(String::from_utf8_unchecked(output)) } } @@ -97,13 +113,13 @@ where /// You can pass either a [`HashMap`][std::collections::HashMap], [`BTreeMap`][std::collections::BTreeMap] as the `variables` parameter. /// The maps must have [`&str`] or [`String`] keys, and the values must be [`AsRef<[u8]>`]. /// On Unix platforms, you can also use [`EnvBytes`]. -pub fn substitute_bytes<'a, M>(source: &[u8], variables: &'a M) -> Result, Error> +pub fn substitute_bytes<'a, M>(source: &[u8], variables: &'a M, mode: Mode) -> Result, Error> where M: VariableMap<'a> + ?Sized, M::Value: AsRef<[u8]>, { let mut output = Vec::with_capacity(source.len() + source.len() / 10); - substitute_impl(&mut output, source, 0..source.len(), variables, &|x| x.as_ref())?; + substitute_impl(&mut output, source, 0..source.len(), variables, &|x| x.as_ref(), mode)?; Ok(output) } @@ -117,6 +133,7 @@ fn substitute_impl<'a, M, F>( range: std::ops::Range, variables: &'a M, to_bytes: &F, + mode: Mode, ) -> Result<(), Error> where M: VariableMap<'a> + ?Sized, @@ -138,17 +155,23 @@ where let value = variables.get(variable.name); match (&value, &variable.default) { (None, None) => { - return Err(error::NoSuchVariable { - position: variable.name_start, - name: variable.name.to_owned(), - } - .into()); + match mode { + Mode::Strict => { + return Err(error::NoSuchVariable { + position: variable.name_start, + name: variable.name.to_owned(), + } + .into()); + }, + Mode::Partial => output.extend_from_slice(&source[next..variable.end_position]), + Mode::Permissive => (), + }; }, (Some(value), _default) => { output.extend_from_slice(to_bytes(value)); }, (None, Some(default)) => { - substitute_impl(output, source, default.clone(), variables, to_bytes)?; + substitute_impl(output, source, default.clone(), variables, to_bytes, mode)?; }, }; finger = variable.end_position; @@ -267,9 +290,9 @@ fn parse_braced_variable(source: &[u8], finger: usize) -> Result error::CharOrByte { } } -/// Find the first non-escaped occurrence of a character. -fn find_non_escaped(needle: u8, haystack: &[u8]) -> Option { +/// Find the closing brace of recursive substitutions. +fn find_closing_brace(haystack: &[u8]) -> Option { let mut finger = 0; + // We need to count the first opening brace + let mut nested = -1; while finger < haystack.len() { - let candidate = memchr::memchr2(b'\\', needle, &haystack[finger..])?; + let candidate = memchr::memchr3(b'\\', b'{', b'}', &haystack[finger..])?; if haystack[finger + candidate] == b'\\' { if candidate == haystack.len() - 1 { return None; } finger += candidate + 2; + } else if haystack[finger + candidate] == b'{' { + if candidate == haystack.len() - 1 { + return None; + } + nested += 1; + finger += candidate + 1; + } else if nested != 0 { + nested -= 1; + finger += candidate + 1; } else { return Some(finger + candidate); } @@ -362,6 +396,7 @@ fn unescape_one(source: &[u8], position: usize) -> Result { #[rustfmt::skip] mod test { use super::*; + use super::Mode::*; use assert2::{assert, check, let_assert}; use std::collections::BTreeMap; @@ -392,52 +427,81 @@ mod test { } #[test] - fn test_find_non_escaped() { - check!(find_non_escaped(b'$', b"$foo") == Some(0)); - check!(find_non_escaped(b'$', b"\\$foo$") == Some(5)); - check!(find_non_escaped(b'$', b"foo $bar") == Some(4)); - check!(find_non_escaped(b'$', b"foo \\$$bar") == Some(6)); + fn test_find_closing_brace() { + check!(find_closing_brace(b"${foo}") == Some(5)); + check!(find_closing_brace(b"{\\{}foo") == Some(3)); + check!(find_closing_brace(b"{{}}foo $bar") == Some(3)); + check!(find_closing_brace(b"foo{\\}}bar") == Some(6)); } #[test] fn test_substitute() { let mut map: BTreeMap = BTreeMap::new(); map.insert("name".into(), "world".into()); - check!(let Ok("Hello world!") = substitute("Hello $name!", &map).as_deref()); - check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map).as_deref()); - check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map).as_deref()); - check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map).as_deref()); + // Strinct + for mode in [Strict,Partial,Permissive]{ + check!(let Ok("Hello world!") = substitute("Hello $name!", &map, mode).as_deref()); + check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map, mode).as_deref()); + check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map, mode).as_deref()); + check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map, mode).as_deref()); + } let mut map: BTreeMap<&str, &str> = BTreeMap::new(); map.insert("name", "world"); - check!(let Ok("Hello world!") = substitute("Hello $name!", &map).as_deref()); - check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map).as_deref()); - check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map).as_deref()); - check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map).as_deref()); + for mode in [Strict,Partial,Permissive]{ + check!(let Ok("Hello world!") = substitute("Hello $name!", &map,mode).as_deref()); + check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map,mode).as_deref()); + check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map,mode).as_deref()); + check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map,mode).as_deref()); + } + } + + #[test] + fn test_substitute_non_strict(){ + let map: BTreeMap = BTreeMap::new(); + check!(let Ok("Hello $name!") = substitute("Hello $name!", &map, Partial).as_deref()); + check!(let Ok("Hello ${name}!") = substitute("Hello ${name}!", &map, Partial).as_deref()); + check!(let Ok("Hello !") = substitute("Hello $name!", &map, Permissive).as_deref()); + check!(let Ok("Hello !") = substitute("Hello ${name}!", &map, Permissive).as_deref()); } #[test] fn substitution_in_default_value() { let mut map: BTreeMap = BTreeMap::new(); + check!(let Ok("Hello cruel $name!") = substitute("Hello ${not_name:cruel $name}!", &map, Partial).as_deref()); + check!(let Ok("Hello cruel !") = substitute("Hello ${not_name:cruel $name}!", &map, Permissive).as_deref()); map.insert("name".into(), "world".into()); - check!(let Ok("Hello cruel world!") = substitute("Hello ${not_name:cruel $name}!", &map).as_deref()); + for mode in [Strict,Partial,Permissive]{ + check!(let Ok("Hello cruel world!") = substitute("Hello ${not_name:cruel $name}!", &map, mode).as_deref()); + } + } + + #[test] + fn recursive_substitution_in_default_value() { + let map: BTreeMap = BTreeMap::new(); + for mode in [Strict,Partial,Permissive]{ + check!(let Ok("Hello cruel world!") = substitute("Hello ${not_name:cruel ${name:world}}!", &map, mode).as_deref()); + } } #[test] fn test_substitute_bytes() { let mut map: BTreeMap> = BTreeMap::new(); map.insert("name".into(), b"world"[..].into()); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map).as_deref()); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map).as_deref()); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map).as_deref()); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map).as_deref()); - + for mode in [Strict,Partial,Permissive]{ + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map, mode).as_deref()); + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map, mode).as_deref()); + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map, mode).as_deref()); + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map, mode).as_deref()); + } let mut map: BTreeMap<&str, &[u8]> = BTreeMap::new(); map.insert("name", b"world"); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map).as_deref()); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map).as_deref()); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map).as_deref()); - check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map).as_deref()); + for mode in [Strict,Partial,Permissive]{ + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map, mode).as_deref()); + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map, mode).as_deref()); + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map, mode).as_deref()); + check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map, mode).as_deref()); + } } #[test] @@ -445,7 +509,7 @@ mod test { let map: BTreeMap = BTreeMap::new(); let source = r"Hello \world!"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map,Strict)); assert!(e.to_string() == r"Invalid escape sequence: \w"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -454,7 +518,7 @@ mod test { )); let source = r"Hello \❤❤"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == r"Invalid escape sequence: \❤"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -463,7 +527,7 @@ mod test { )); let source = r"Hello world!\"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == r"Invalid escape sequence: missing escape character"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -477,7 +541,7 @@ mod test { let map: BTreeMap = BTreeMap::new(); let source = r"Hello $!"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == r"Missing variable name"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -486,7 +550,7 @@ mod test { )); let source = r"Hello ${}!"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == r"Missing variable name"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -495,7 +559,7 @@ mod test { )); let source = r"Hello ${:fallback}!"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == r"Missing variable name"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -504,7 +568,7 @@ mod test { )); let source = r"Hello  $❤"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == r"Missing variable name"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -518,7 +582,7 @@ mod test { let map: BTreeMap = BTreeMap::new(); let source = "Hello ${name)!"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == "Unexpected character: ')', expected a closing brace ('}') or colon (':')"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -527,7 +591,7 @@ mod test { )); let source = "Hello ${name❤"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == "Unexpected character: '❤', expected a closing brace ('}') or colon (':')"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -536,7 +600,7 @@ mod test { )); let source = b"\xE2\x98Hello ${name\xE2\x98"; - let_assert!(Err(e) = substitute_bytes(source, &map)); + let_assert!(Err(e) = substitute_bytes(source, &map, Strict)); assert!(e.to_string() == "Unexpected character: '\\xE2', expected a closing brace ('}') or colon (':')"); } @@ -545,7 +609,7 @@ mod test { let map: BTreeMap = BTreeMap::new(); let source = "Hello ${name"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == "Missing closing brace"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -554,7 +618,7 @@ mod test { )); let source = "Hello ${name:fallback"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == "Missing closing brace"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -568,7 +632,7 @@ mod test { let map: BTreeMap = BTreeMap::new(); let source = "Hello ${name}!"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == "No such variable: $name"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -577,7 +641,7 @@ mod test { )); let source = "Hello $name!"; - let_assert!(Err(e) = substitute(source, &map)); + let_assert!(Err(e) = substitute(source, &map, Strict)); assert!(e.to_string() == "No such variable: $name"); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( @@ -592,7 +656,7 @@ mod test { variables.insert(String::from("aap"), String::from("noot")); let variables: &dyn VariableMap = &variables; - let_assert!(Ok(expanded) = substitute("one ${aap}", variables)); + let_assert!(Ok(expanded) = substitute("one ${aap}", variables, Strict)); assert!(expanded == "one noot"); } @@ -602,7 +666,7 @@ mod test { variables.insert(String::from("aap"), String::from("noot")); let source = r"emoticon: \( ^▽^ )/"; - let_assert!(Err(e) = substitute(source, &variables)); + let_assert!(Err(e) = substitute(source, &variables, Strict)); #[rustfmt::skip] assert!(e.source_highlighting(source) == concat!( r" emoticon: \( ^▽^ )/", "\n", diff --git a/src/yaml.rs b/src/yaml.rs index edf8792..95277a4 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -2,6 +2,7 @@ use serde::de::DeserializeOwned; +use crate::Mode; use crate::VariableMap; /// Parse a struct from YAML data, after performing variable substitution on string values. @@ -9,13 +10,13 @@ use crate::VariableMap; /// This function first parses the data into a [`serde_yaml::Value`], /// then performs variable substitution on all string values, /// and then parses it further into the desired type. -pub fn from_slice<'a, T: DeserializeOwned, M>(data: &[u8], variables: &'a M) -> Result +pub fn from_slice<'a, T: DeserializeOwned, M>(data: &[u8], variables: &'a M, mode: Mode) -> Result where M: VariableMap<'a> + ?Sized, M::Value: AsRef, { let mut value: serde_yaml::Value = serde_yaml::from_slice(data)?; - substitute_string_values(&mut value, variables)?; + substitute_string_values(&mut value, variables, mode)?; Ok(serde_yaml::from_value(value)?) } @@ -24,24 +25,28 @@ where /// This function first parses the data into a [`serde_yaml::Value`], /// then performs variable substitution on all string values, /// and then parses it further into the desired type. -pub fn from_str<'a, T: DeserializeOwned, M>(data: &str, variables: &'a M) -> Result +pub fn from_str<'a, T: DeserializeOwned, M>(data: &str, variables: &'a M, mode: Mode) -> Result where M: VariableMap<'a> + ?Sized, M::Value: AsRef, { let mut value: serde_yaml::Value = serde_yaml::from_str(data)?; - substitute_string_values(&mut value, variables)?; + substitute_string_values(&mut value, variables, mode)?; Ok(serde_yaml::from_value(value)?) } /// Perform variable substitution on string values of a YAML value. -pub fn substitute_string_values<'a, M>(value: &mut serde_yaml::Value, variables: &'a M) -> Result<(), crate::Error> +pub fn substitute_string_values<'a, M>( + value: &mut serde_yaml::Value, + variables: &'a M, + mode: Mode, +) -> Result<(), crate::Error> where M: VariableMap<'a> + ?Sized, M::Value: AsRef, { visit_string_values(value, |value| { - *value = crate::substitute(value.as_str(), variables)?; + *value = crate::substitute(value.as_str(), variables, mode)?; Ok(()) }) } @@ -114,6 +119,7 @@ mod test { use std::collections::HashMap; use super::*; + use super::Mode::*; use assert2::{assert, let_assert}; #[test] @@ -133,7 +139,7 @@ mod test { "bar: $bar\n", "baz: $baz/with/stuff\n", ), - &variables, + &variables, Strict )); let parsed: Struct = parsed; @@ -158,7 +164,7 @@ mod test { "bar: aap\n", "baz: noot/with/stuff\n", ), - &crate::NoSubstitution, + &crate::NoSubstitution, Strict )); let parsed: Struct = parsed; @@ -183,7 +189,7 @@ mod test { "bar: $bar\n", "baz: $baz\n", ), - &variables, + &variables, Strict )); let parsed: Struct = parsed; @@ -208,7 +214,7 @@ mod test { "bar: !!string $bar\n", "baz: $baz\n", ), - &variables, + &variables, Strict )); let parsed: Struct = parsed; @@ -234,7 +240,7 @@ mod test { "bar: $bar\n", "baz: $baz/with/stuff\n", ), - variables, + variables, Strict )); let parsed: Struct = parsed;