feat(locations): interpolate environment variables in YAML input (#158)

Expand ${VAR} / ${VAR:-default} / ${VAR:?msg} in requirements.yml,
svcs.yml, mvrs.yml, annotations.yml and reqstool_config.yml before
parsing, using POSIX shell parameter expansion (the envsubst standard)
via the expandvars library.

Only the braced form is interpolated; the bare $VAR form is left
untouched so that the '# yaml-language-server: $schema=...' directive,
regexes and other literal $ characters survive unchanged. An unset
variable without an inline default is a hard error, keeping ingestion
deterministic and CI-safe.

This is the principled, deterministic alternative to 'version: latest'
(#138): pin versions to env vars and let Renovate keep them current.

Round-trip (typ=rt) loads backing LSP features are intentionally not
interpolated, so editor positions are unaffected.

Signed-off-by: Jimisola Laursen <jimisola@jimisola.com>

Author: jimisola

docs/modules/ROOT/pages/usage.adoc

@@ -22,6 +22,44 @@ Each command accepts a location argument that specifies where the reqstool data
 reqstool report git -u URL_TO_REPOSITORY.git -t ACCESS_TOKEN -p PATH_TO_DIR -r REF
 ----
 
+[[env-var-interpolation]]
+== Environment variable interpolation
+
+Reqstool expands environment variables in YAML input (`requirements.yml`, `svcs.yml`,
+`mvrs.yml`, `annotations.yml`, `reqstool_config.yml`) before parsing, using POSIX shell
+parameter expansion (the `envsubst` standard). Only the *braced* form is interpolated:
+
+[cols="1,3"]
+|===
+| Syntax | Behaviour
+
+| `${VAR}`            | Substitute the value of `VAR`.
+| `${VAR:-default}`   | Use `default` when `VAR` is unset or empty.
+| `${VAR:?message}`   | Fail with `message` when `VAR` is unset or empty.
+| `${VAR:+alt}`       | Use `alt` when `VAR` is set.
+|===
+
+A bare `${VAR}` whose variable is unset and has no inline default is a *hard error* -- this
+keeps ingestion deterministic and surfaces misconfiguration in CI rather than silently
+producing empty values.
+
+The bare `$VAR` form (without braces) is intentionally *not* expanded, so the
+`# yaml-language-server: $schema=...` directive, regular expressions and other literal `$`
+characters are left untouched.
+
+This is the recommended way to keep imported artifact versions current -- pin the version to
+an environment variable and let a tool such as Renovate update it:
+
+[source,yaml]
+----
+imports:
+  maven:
+    - url: https://repo.maven.org
+      group_id: com.example
+      artifact_id: my-lib
+      version: ${MY_LIB_VERSION}
+----
+
 [[status]]
 == Command: status
 

pyproject.toml

@@ -53,6 +53,7 @@ dependencies = [
     "packaging==26.2",
     "requests==2.34.2",
     "beautifulsoup4==4.14.3",
+    "expandvars==1.1.2",
     "pygls>=2.0,<3.0",
     "lsprotocol>=2024.0.0",
     "mcp>=1.0",

src/reqstool/common/exceptions.py

@@ -46,3 +46,16 @@ def __init__(self, ref: str, url: str):
         self.ref = ref
         self.url = url
         super().__init__(f"ref '{ref}' not found in {url}")
+
+
+class EnvVarInterpolationError(Exception):
+    """Raised when environment variable interpolation of YAML input fails.
+
+    This covers references to unset variables that have no inline default, as
+    well as malformed ``${...}`` expressions. Failing hard keeps ingestion
+    deterministic and surfaces configuration mistakes in CI.
+    """
+
+    def __init__(self, message: str, source: str | None = None):
+        self.source = source
+        super().__init__(f"{message} (in {source})" if source else message)

src/reqstool/common/utils.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import re
 import tarfile
 import tempfile
 from importlib.metadata import version
@@ -10,10 +11,12 @@
 from typing import Dict, Iterable, List, Sequence
 from zipfile import ZipFile
 
+import expandvars
 import requests
 from packaging.version import InvalidVersion, Version as PkgVersion
 from requests_file import FileAdapter
 
+from reqstool.common.exceptions import EnvVarInterpolationError
 from reqstool.common.models.urn_id import UrnId
 from reqstool.models.raw_datasets import RawDataset
 from reqstool.models.requirements import RequirementData
@@ -24,6 +27,45 @@ class Utils:
 
     is_installed_package: bool = True
 
+    # Only the braced ``${...}`` form is interpolated. The bare ``$VAR`` form is
+    # intentionally left untouched: reqstool YAML files routinely contain a
+    # ``# yaml-language-server: $schema=...`` directive (and regexes, prices,
+    # etc.) where a stray ``$`` must not be treated as a variable reference.
+    _ENV_VAR_PATTERN = re.compile(r"\$\{[^{}]*\}")
+
+    @staticmethod
+    def interpolate_env_vars(text: str, source: str | None = None) -> str:
+        """Expand environment variables in raw YAML text before parsing.
+
+        Uses POSIX shell parameter expansion (the ``envsubst`` standard),
+        restricted to the braced form::
+
+            ${VAR}            substitute the value of VAR
+            ${VAR:-default}   use ``default`` when VAR is unset or empty
+            ${VAR:?message}   fail with ``message`` when VAR is unset or empty
+            ${VAR:+alt}       use ``alt`` when VAR is set
+
+        A bare ``${VAR}`` whose variable is unset (and has no inline default)
+        is a hard error: this keeps ingestion deterministic and catches
+        misconfiguration in CI rather than silently producing empty values.
+
+        Args:
+            text: raw file contents to interpolate.
+            source: optional file path/URI used in error messages.
+
+        Raises:
+            EnvVarInterpolationError: on an unset variable without a default or
+                a malformed ``${...}`` expression.
+        """
+
+        def _expand(match: "re.Match[str]") -> str:
+            try:
+                return expandvars.expand(match.group(0), nounset=True)
+            except expandvars.ExpandvarsException as e:
+                raise EnvVarInterpolationError(str(e), source=source) from e
+
+        return Utils._ENV_VAR_PATTERN.sub(_expand, text)
+
     @staticmethod
     def get_version() -> str:
         ver: str = f"{version('reqstool')}" if Utils.is_installed_package else "local-dev"

src/reqstool/model_generators/annotations_model_generator.py

@@ -24,7 +24,7 @@ def __generate(self, uri: str) -> AnnotationsData:
 
         yaml = YAML(typ="safe")
 
-        data: dict = yaml.load(response.text)
+        data: dict = yaml.load(Utils.interpolate_env_vars(response.text, source=uri))
 
         if not SyntaxValidator.is_valid_data(json_schema_type=JsonSchemaTypes.ANNOTATIONS, data=data, urn=self.urn):
             sys.exit(EXIT_CODE_SYNTAX_VALIDATION_ERROR)

src/reqstool/model_generators/mvrs_model_generator.py

@@ -26,7 +26,7 @@ def __generate(self, uri: str) -> MVRsData:
 
         yaml = YAML(typ="safe")
 
-        data: dict = yaml.load(response.text)
+        data: dict = yaml.load(Utils.interpolate_env_vars(response.text, source=uri))
 
         if not SyntaxValidator.is_valid_data(
             json_schema_type=JsonSchemaTypes.MANUAL_VERIFICATION_RESULTS, data=data, urn=self.urn

src/reqstool/model_generators/requirements_model_generator.py

@@ -87,7 +87,7 @@ def __generate(
 
         yaml = YAML(typ="safe")
 
-        data = yaml.load(response.text)
+        data = yaml.load(Utils.interpolate_env_vars(response.text, source=uri))
 
         urn = self.get_urn_if_available(response.text)
 

src/reqstool/model_generators/svcs_model_generator.py

@@ -37,7 +37,7 @@ def __generate(self, uri: str) -> SVCsData:
 
         yaml = YAML(typ="safe")
 
-        data: dict = yaml.load(response.text)
+        data: dict = yaml.load(Utils.interpolate_env_vars(response.text, source=uri))
 
         if not SyntaxValidator.is_valid_data(
             json_schema_type=JsonSchemaTypes.SOFTWARE_VERIFICATION_CASES, data=data, urn=self.urn

src/reqstool/requirements_indata/requirements_indata.py

@@ -41,11 +41,12 @@ def model_post_init(self, __context):
     def _handle_requirements_config(self):
 
         if os.path.exists(os.path.join(self.dst_path, "reqstool_config.yml")):
-            response = Utils.open_file_https_file(os.path.join(self.dst_path, "reqstool_config.yml"))
+            config_path = os.path.join(self.dst_path, "reqstool_config.yml")
+            response = Utils.open_file_https_file(config_path)
 
             yaml = YAML(typ="safe")
 
-            data: dict = yaml.load(response.text)
+            data: dict = yaml.load(Utils.interpolate_env_vars(response.text, source=config_path))
 
             if not SyntaxValidator.is_valid_data(
                 json_schema_type=JsonSchemaTypes.REQSTOOL_CONFIG, data=data, urn="unknown"

tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_requirements_model_generator_interpolates_env_vars/requirements.yml

@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/reqstool/reqstool-client/main/src/reqstool/resources/schemas/v1/requirements.schema.json
+
+metadata:
+  urn: sys-001
+  variant: system
+  title: ${REQSTOOL_TEST_TITLE:-Default Title}
+
+imports:
+  maven:
+    - url: https://repo.maven.org
+      group_id: com.example.one
+      artifact_id: test-one
+      version: ${REQSTOOL_TEST_MAVEN_VERSION}
+
+requirements:
+  - id: REQ_001
+    title: Title REQ_001
+    significance: may
+    description: Description REQ_001
+    rationale: Rationale REQ_001
+    categories: ["maintainability"]
+    revision: 0.0.1