Skip to content

feat!: Add partial and permissive substitutions and recursive braced variables substitution #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
+142 −72
Draft

feat!: Add partial and permissive substitutions and recursive braced variables substitution #18

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
20d2c20
feat!: Add partial and permissive substitutions
D-Brox May 1, 2024
bf000c5
fix: Allow recursive substitution of braced variables
D-Brox May 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 125 additions & 61 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@
//! # use std::collections::HashMap;
//! let mut variables = HashMap::new();
//! variables.insert("name", "world");
//! assert_eq!(subst::substitute("Hello $name!", &variables)?, "Hello world!");
//! assert_eq!(subst::substitute("Hello $name!", &variables, subst::Mode::Strict)?, "Hello world!");
//! # Ok(())
//! # }
//! ```
@@ -34,8 +34,8 @@
//! # fn main() -> Result<(), subst::Error> {
//! # std::env::set_var("XDG_CONFIG_HOME", "/home/user/.config");
//! assert_eq!(
//! subst::substitute("$XDG_CONFIG_HOME/my-app/config.toml", &subst::Env)?,
//! "/home/user/.config/my-app/config.toml",
//! subst::substitute("$XDG_CONFIG_HOME/my-app/config.toml", &subst::Env, subst::Mode::Strict)?,
//! "/home/user/.config/my-app/config.toml"
//! );
//! # Ok(())
//! # }
@@ -48,7 +48,7 @@
//! # use std::collections::HashMap;
//! let mut variables = HashMap::new();
//! variables.insert("name", b"world");
//! assert_eq!(subst::substitute_bytes(b"Hello $name!", &variables)?, b"Hello world!");
//! assert_eq!(subst::substitute_bytes(b"Hello $name!", &variables, subst::Mode::Strict)?, b"Hello world!");
//! # Ok(())
//! # }
//! ```
@@ -63,6 +63,17 @@ pub use map::*;
#[cfg(feature = "yaml")]
pub mod yaml;

/// Substitution mode
#[derive(Debug, Copy, Clone)]
pub enum Mode {
/// Do not allow missing variables.
Strict,
/// Do not substitute missing variables without default.
Partial,
/// Substitute missing variables with and empty string.
Permissive,
}

/// Substitute variables in a string.
///
/// Variables have the form `$NAME`, `${NAME}` or `${NAME:default}`.
@@ -73,15 +84,20 @@ pub mod yaml;
///
/// You can pass either a [`HashMap`][std::collections::HashMap], [`BTreeMap`][std::collections::BTreeMap] or [`Env`] as the `variables` parameter.
/// The maps must have [`&str`] or [`String`] keys, and the values must be [`AsRef<str>`].
pub fn substitute<'a, M>(source: &str, variables: &'a M) -> Result<String, Error>
pub fn substitute<'a, M>(source: &str, variables: &'a M, mode: Mode) -> Result<String, Error>
where
M: VariableMap<'a> + ?Sized,
M::Value: AsRef<str>,
{
let mut output = Vec::with_capacity(source.len() + source.len() / 10);
substitute_impl(&mut output, source.as_bytes(), 0..source.len(), variables, &|x| {
x.as_ref().as_bytes()
})?;
substitute_impl(
&mut output,
source.as_bytes(),
0..source.len(),
variables,
&|x| x.as_ref().as_bytes(),
mode,
)?;
// SAFETY: Both source and all variable values are valid UTF-8, so substitation result is also valid UTF-8.
unsafe { Ok(String::from_utf8_unchecked(output)) }
}
@@ -97,13 +113,13 @@ where
/// You can pass either a [`HashMap`][std::collections::HashMap], [`BTreeMap`][std::collections::BTreeMap] as the `variables` parameter.
/// The maps must have [`&str`] or [`String`] keys, and the values must be [`AsRef<[u8]>`].
/// On Unix platforms, you can also use [`EnvBytes`].
pub fn substitute_bytes<'a, M>(source: &[u8], variables: &'a M) -> Result<Vec<u8>, Error>
pub fn substitute_bytes<'a, M>(source: &[u8], variables: &'a M, mode: Mode) -> Result<Vec<u8>, Error>
where
M: VariableMap<'a> + ?Sized,
M::Value: AsRef<[u8]>,
{
let mut output = Vec::with_capacity(source.len() + source.len() / 10);
substitute_impl(&mut output, source, 0..source.len(), variables, &|x| x.as_ref())?;
substitute_impl(&mut output, source, 0..source.len(), variables, &|x| x.as_ref(), mode)?;
Ok(output)
}

@@ -117,6 +133,7 @@ fn substitute_impl<'a, M, F>(
range: std::ops::Range<usize>,
variables: &'a M,
to_bytes: &F,
mode: Mode,
) -> Result<(), Error>
where
M: VariableMap<'a> + ?Sized,
@@ -138,17 +155,23 @@ where
let value = variables.get(variable.name);
match (&value, &variable.default) {
(None, None) => {
return Err(error::NoSuchVariable {
position: variable.name_start,
name: variable.name.to_owned(),
}
.into());
match mode {
Mode::Strict => {
return Err(error::NoSuchVariable {
position: variable.name_start,
name: variable.name.to_owned(),
}
.into());
},
Mode::Partial => output.extend_from_slice(&source[next..variable.end_position]),
Mode::Permissive => (),
};
},
(Some(value), _default) => {
output.extend_from_slice(to_bytes(value));
},
(None, Some(default)) => {
substitute_impl(output, source, default.clone(), variables, to_bytes)?;
substitute_impl(output, source, default.clone(), variables, to_bytes, mode)?;
},
};
finger = variable.end_position;
@@ -267,9 +290,9 @@ fn parse_braced_variable(source: &[u8], finger: usize) -> Result<Variable, Error
.into());
}

// If there is no un-escaped closing brace, it's missing.
let end = finger
+ find_non_escaped(b'}', &source[finger..]).ok_or(error::MissingClosingBrace { position: finger + 1 })?;
// If there is no un-escaped closing brace pair, it's missing.
let end =
finger + find_closing_brace(&source[finger..]).ok_or(error::MissingClosingBrace { position: finger + 1 })?;

Ok(Variable {
name: std::str::from_utf8(&source[name_start..name_end]).unwrap(),
@@ -313,16 +336,27 @@ fn get_maybe_char_at(data: &[u8], index: usize) -> error::CharOrByte {
}
}

/// Find the first non-escaped occurrence of a character.
fn find_non_escaped(needle: u8, haystack: &[u8]) -> Option<usize> {
/// Find the closing brace of recursive substitutions.
fn find_closing_brace(haystack: &[u8]) -> Option<usize> {
let mut finger = 0;
// We need to count the first opening brace
let mut nested = -1;
while finger < haystack.len() {
let candidate = memchr::memchr2(b'\\', needle, &haystack[finger..])?;
let candidate = memchr::memchr3(b'\\', b'{', b'}', &haystack[finger..])?;
if haystack[finger + candidate] == b'\\' {
if candidate == haystack.len() - 1 {
return None;
}
finger += candidate + 2;
} else if haystack[finger + candidate] == b'{' {
if candidate == haystack.len() - 1 {
return None;
}
nested += 1;
finger += candidate + 1;
} else if nested != 0 {
nested -= 1;
finger += candidate + 1;
} else {
return Some(finger + candidate);
}
@@ -362,6 +396,7 @@ fn unescape_one(source: &[u8], position: usize) -> Result<u8, Error> {
#[rustfmt::skip]
mod test {
use super::*;
use super::Mode::*;
use assert2::{assert, check, let_assert};
use std::collections::BTreeMap;

@@ -392,60 +427,89 @@ mod test {
}

#[test]
fn test_find_non_escaped() {
check!(find_non_escaped(b'$', b"$foo") == Some(0));
check!(find_non_escaped(b'$', b"\\$foo$") == Some(5));
check!(find_non_escaped(b'$', b"foo $bar") == Some(4));
check!(find_non_escaped(b'$', b"foo \\$$bar") == Some(6));
fn test_find_closing_brace() {
check!(find_closing_brace(b"${foo}") == Some(5));
check!(find_closing_brace(b"{\\{}foo") == Some(3));
check!(find_closing_brace(b"{{}}foo $bar") == Some(3));
check!(find_closing_brace(b"foo{\\}}bar") == Some(6));
}

#[test]
fn test_substitute() {
let mut map: BTreeMap<String, String> = BTreeMap::new();
map.insert("name".into(), "world".into());
check!(let Ok("Hello world!") = substitute("Hello $name!", &map).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map).as_deref());
// Strinct
for mode in [Strict,Partial,Permissive]{
check!(let Ok("Hello world!") = substitute("Hello $name!", &map, mode).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map, mode).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map, mode).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map, mode).as_deref());
}

let mut map: BTreeMap<&str, &str> = BTreeMap::new();
map.insert("name", "world");
check!(let Ok("Hello world!") = substitute("Hello $name!", &map).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map).as_deref());
for mode in [Strict,Partial,Permissive]{
check!(let Ok("Hello world!") = substitute("Hello $name!", &map,mode).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name}!", &map,mode).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${name:not-world}!", &map,mode).as_deref());
check!(let Ok("Hello world!") = substitute("Hello ${not_name:world}!", &map,mode).as_deref());
}
}

#[test]
fn test_substitute_non_strict(){
let map: BTreeMap<String, String> = BTreeMap::new();
check!(let Ok("Hello $name!") = substitute("Hello $name!", &map, Partial).as_deref());
check!(let Ok("Hello ${name}!") = substitute("Hello ${name}!", &map, Partial).as_deref());
check!(let Ok("Hello !") = substitute("Hello $name!", &map, Permissive).as_deref());
check!(let Ok("Hello !") = substitute("Hello ${name}!", &map, Permissive).as_deref());
}

#[test]
fn substitution_in_default_value() {
let mut map: BTreeMap<String, String> = BTreeMap::new();
check!(let Ok("Hello cruel $name!") = substitute("Hello ${not_name:cruel $name}!", &map, Partial).as_deref());
check!(let Ok("Hello cruel !") = substitute("Hello ${not_name:cruel $name}!", &map, Permissive).as_deref());
map.insert("name".into(), "world".into());
check!(let Ok("Hello cruel world!") = substitute("Hello ${not_name:cruel $name}!", &map).as_deref());
for mode in [Strict,Partial,Permissive]{
check!(let Ok("Hello cruel world!") = substitute("Hello ${not_name:cruel $name}!", &map, mode).as_deref());
}
}

#[test]
fn recursive_substitution_in_default_value() {
let map: BTreeMap<String, String> = BTreeMap::new();
for mode in [Strict,Partial,Permissive]{
check!(let Ok("Hello cruel world!") = substitute("Hello ${not_name:cruel ${name:world}}!", &map, mode).as_deref());
}
}

#[test]
fn test_substitute_bytes() {
let mut map: BTreeMap<String, Vec<u8>> = BTreeMap::new();
map.insert("name".into(), b"world"[..].into());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map).as_deref());

for mode in [Strict,Partial,Permissive]{
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map, mode).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map, mode).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map, mode).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map, mode).as_deref());
}
let mut map: BTreeMap<&str, &[u8]> = BTreeMap::new();
map.insert("name", b"world");
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map).as_deref());
for mode in [Strict,Partial,Permissive]{
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello $name!", &map, mode).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name}!", &map, mode).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${name:not-world}!", &map, mode).as_deref());
check!(let Ok(b"Hello world!") = substitute_bytes(b"Hello ${not_name:world}!", &map, mode).as_deref());
}
}

#[test]
fn test_invalid_escape_sequence() {
let map: BTreeMap<String, String> = BTreeMap::new();

let source = r"Hello \world!";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map,Strict));
assert!(e.to_string() == r"Invalid escape sequence: \w");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -454,7 +518,7 @@ mod test {
));

let source = r"Hello \❤❤";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == r"Invalid escape sequence: \❤");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -463,7 +527,7 @@ mod test {
));

let source = r"Hello world!\";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == r"Invalid escape sequence: missing escape character");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -477,7 +541,7 @@ mod test {
let map: BTreeMap<String, String> = BTreeMap::new();

let source = r"Hello $!";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == r"Missing variable name");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -486,7 +550,7 @@ mod test {
));

let source = r"Hello ${}!";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == r"Missing variable name");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -495,7 +559,7 @@ mod test {
));

let source = r"Hello ${:fallback}!";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == r"Missing variable name");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -504,7 +568,7 @@ mod test {
));

let source = r"Hello 　$❤";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == r"Missing variable name");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -518,7 +582,7 @@ mod test {
let map: BTreeMap<String, String> = BTreeMap::new();

let source = "Hello ${name)!";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == "Unexpected character: ')', expected a closing brace ('}') or colon (':')");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -527,7 +591,7 @@ mod test {
));

let source = "Hello ${name❤";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == "Unexpected character: '❤', expected a closing brace ('}') or colon (':')");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -536,7 +600,7 @@ mod test {
));

let source = b"\xE2\x98Hello ${name\xE2\x98";
let_assert!(Err(e) = substitute_bytes(source, &map));
let_assert!(Err(e) = substitute_bytes(source, &map, Strict));
assert!(e.to_string() == "Unexpected character: '\\xE2', expected a closing brace ('}') or colon (':')");
}

@@ -545,7 +609,7 @@ mod test {
let map: BTreeMap<String, String> = BTreeMap::new();

let source = "Hello ${name";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == "Missing closing brace");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -554,7 +618,7 @@ mod test {
));

let source = "Hello ${name:fallback";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == "Missing closing brace");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -568,7 +632,7 @@ mod test {
let map: BTreeMap<String, String> = BTreeMap::new();

let source = "Hello ${name}!";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == "No such variable: $name");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -577,7 +641,7 @@ mod test {
));

let source = "Hello $name!";
let_assert!(Err(e) = substitute(source, &map));
let_assert!(Err(e) = substitute(source, &map, Strict));
assert!(e.to_string() == "No such variable: $name");
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
@@ -592,7 +656,7 @@ mod test {
variables.insert(String::from("aap"), String::from("noot"));
let variables: &dyn VariableMap<Value = &String> = &variables;

let_assert!(Ok(expanded) = substitute("one ${aap}", variables));
let_assert!(Ok(expanded) = substitute("one ${aap}", variables, Strict));
assert!(expanded == "one noot");
}

@@ -602,7 +666,7 @@ mod test {
variables.insert(String::from("aap"), String::from("noot"));

let source = r"emoticon: \（ ^▽^ ）/";
let_assert!(Err(e) = substitute(source, &variables));
let_assert!(Err(e) = substitute(source, &variables, Strict));
#[rustfmt::skip]
assert!(e.source_highlighting(source) == concat!(
r" emoticon: \（ ^▽^ ）/", "\n",
28 changes: 17 additions & 11 deletions src/yaml.rs
View file
Original file line number Diff line number Diff line change
@@ -2,20 +2,21 @@
use serde::de::DeserializeOwned;

use crate::Mode;
use crate::VariableMap;

/// Parse a struct from YAML data, after performing variable substitution on string values.
///
/// This function first parses the data into a [`serde_yaml::Value`],
/// then performs variable substitution on all string values,
/// and then parses it further into the desired type.
pub fn from_slice<'a, T: DeserializeOwned, M>(data: &[u8], variables: &'a M) -> Result<T, Error>
pub fn from_slice<'a, T: DeserializeOwned, M>(data: &[u8], variables: &'a M, mode: Mode) -> Result<T, Error>
where
M: VariableMap<'a> + ?Sized,
M::Value: AsRef<str>,
{
let mut value: serde_yaml::Value = serde_yaml::from_slice(data)?;
substitute_string_values(&mut value, variables)?;
substitute_string_values(&mut value, variables, mode)?;
Ok(serde_yaml::from_value(value)?)
}

@@ -24,24 +25,28 @@ where
/// This function first parses the data into a [`serde_yaml::Value`],
/// then performs variable substitution on all string values,
/// and then parses it further into the desired type.
pub fn from_str<'a, T: DeserializeOwned, M>(data: &str, variables: &'a M) -> Result<T, Error>
pub fn from_str<'a, T: DeserializeOwned, M>(data: &str, variables: &'a M, mode: Mode) -> Result<T, Error>
where
M: VariableMap<'a> + ?Sized,
M::Value: AsRef<str>,
{
let mut value: serde_yaml::Value = serde_yaml::from_str(data)?;
substitute_string_values(&mut value, variables)?;
substitute_string_values(&mut value, variables, mode)?;
Ok(serde_yaml::from_value(value)?)
}

/// Perform variable substitution on string values of a YAML value.
pub fn substitute_string_values<'a, M>(value: &mut serde_yaml::Value, variables: &'a M) -> Result<(), crate::Error>
pub fn substitute_string_values<'a, M>(
value: &mut serde_yaml::Value,
variables: &'a M,
mode: Mode,
) -> Result<(), crate::Error>
where
M: VariableMap<'a> + ?Sized,
M::Value: AsRef<str>,
{
visit_string_values(value, |value| {
*value = crate::substitute(value.as_str(), variables)?;
*value = crate::substitute(value.as_str(), variables, mode)?;
Ok(())
})
}
@@ -114,6 +119,7 @@ mod test {
use std::collections::HashMap;

use super::*;
use super::Mode::*;
use assert2::{assert, let_assert};

#[test]
@@ -133,7 +139,7 @@ mod test {
"bar: $bar\n",
"baz: $baz/with/stuff\n",
),
&variables,
&variables, Strict
));

let parsed: Struct = parsed;
@@ -158,7 +164,7 @@ mod test {
"bar: aap\n",
"baz: noot/with/stuff\n",
),
&crate::NoSubstitution,
&crate::NoSubstitution, Strict
));

let parsed: Struct = parsed;
@@ -183,7 +189,7 @@ mod test {
"bar: $bar\n",
"baz: $baz\n",
),
&variables,
&variables, Strict
));

let parsed: Struct = parsed;
@@ -208,7 +214,7 @@ mod test {
"bar: !!string $bar\n",
"baz: $baz\n",
),
&variables,
&variables, Strict
));

let parsed: Struct = parsed;
@@ -234,7 +240,7 @@ mod test {
"bar: $bar\n",
"baz: $baz/with/stuff\n",
),
variables,
variables, Strict
));

let parsed: Struct = parsed;