diff --git a/src/charonload/_config.py b/src/charonload/_config.py index 032502d..e57996c 100644 --- a/src/charonload/_config.py +++ b/src/charonload/_config.py @@ -3,6 +3,7 @@ import base64 import getpass import hashlib +import os import pathlib import re import site @@ -24,9 +25,9 @@ @dataclass(init=False) # Python 3.10+: Use "_: KW_ONLY" class Config: """ - Set of user-specified configuration options required for the import logic of the :class:`JITCompileFinder`. + Set of user-specified configuration options for setting up the import logic of :class:`JITCompileFinder`. - This will be resolved into :class:`ResolvedConfig`. + This will be resolved into :class:`ResolvedConfig` before usage. """ project_directory: pathlib.Path | str @@ -44,6 +45,12 @@ class Config: Whether to remove all cached files of previous builds from the build directory. This is useful to ensure consistent behavior after major changes in the CMake files of the project. + + .. admonition:: Overrides + :class: important + + If the environment variable ``CHARONLOAD_FORCE_CLEAN_BUILD`` is set, it will replace this value in + :class:`ResolvedConfig`. """ build_type: str @@ -62,13 +69,27 @@ class Config: """ stubs_invalid_ok: bool - """Whether to accept invalid stubs and skip raising an error.""" + """ + Whether to accept invalid stubs and skip raising an error. + + .. admonition:: Overrides + :class: important + + If the environment variable ``CHARONLOAD_FORCE_STUBS_INVALID_OK`` is set, it will replace this value in + :class:`ResolvedConfig`. + """ verbose: bool """ Whether to enable printing the full log of the JIT compilation. This is useful for debugging. + + .. admonition:: Overrides + :class: important + + If the environment variable ``CHARONLOAD_FORCE_VERBOSE`` is set, it will replace this value in + :class:`ResolvedConfig`. """ def __init__( @@ -96,9 +117,9 @@ def __init__( @dataclass(init=False) # Python 3.10+: Use "kw_only=True" class ResolvedConfig: """ - Set of resolved configuration options that are actually used for the import logic of the :class:`JITCompileFinder`. + Set of resolved configuration options that is **actually used** in the import logic of :class:`JITCompileFinder`. - This has been resolved from :class:`Config`. + This has been resolved from :class:`Config` and from the environment variables. """ full_project_directory: pathlib.Path @@ -219,17 +240,31 @@ def _resolve(self: Self, module_name: str, config: Config) -> ResolvedConfig: project_directory=config.project_directory, verbose=config.verbose, ), - clean_build=config.clean_build, + clean_build=self._str_to_bool(os.environ.get("CHARONLOAD_FORCE_CLEAN_BUILD", default=config.clean_build)), build_type=config.build_type, cmake_options=config.cmake_options if config.cmake_options is not None else {}, full_stubs_directory=self._find_stubs_directory( stubs_directory=config.stubs_directory, verbose=config.verbose, ), - stubs_invalid_ok=config.stubs_invalid_ok, - verbose=config.verbose, + stubs_invalid_ok=self._str_to_bool( + os.environ.get("CHARONLOAD_FORCE_STUBS_INVALID_OK", default=config.stubs_invalid_ok) + ), + verbose=self._str_to_bool(os.environ.get("CHARONLOAD_FORCE_VERBOSE", default=config.verbose)), ) + def _str_to_bool(self: Self, s: str | bool) -> bool: + if isinstance(s, bool): + return s + + if s.lower() in ["1", "on", "yes", "true", "y"]: + return True + if s.lower() in ["0", "off", "no", "false", "n"]: + return False + + msg = f'Cannot convert string "{s}" to bool' + raise ValueError(msg) + def _find_build_directory( self: Self, *, diff --git a/src/charonload/_errors.py b/src/charonload/_errors.py index 3f12de8..a48a726 100644 --- a/src/charonload/_errors.py +++ b/src/charonload/_errors.py @@ -27,7 +27,25 @@ def __init__(self: Self, step_name: str = "", log: str | None = None) -> None: self.log = log """The full log from the underlying compiler.""" - super().__init__(f"{self.step_name} failed:\n\n{self.log}" if log is not None else f"{self.step_name} failed.") + msg = "" + if self.log is not None: + msg += f"{self.step_name} failed:\n" + msg += "----------------------------------------------------------------\n" + msg += "\n" + msg += self.log + msg += "\n" + msg += "----------------------------------------------------------------\n" + else: + msg += f"{self.step_name} failed.\n" + msg += "\n" + msg += ( + "charonload might automatically run a clean build on the next call in order to try to resolve the error. " + ) + msg += "If the issue persists, you can override charonload's behavior via these environment variables:\n" + msg += " - CHARONLOAD_FORCE_CLEAN_BUILD=1 : Enforce clean builds for all JIT compiled projects.\n" + msg += " - CHARONLOAD_FORCE_VERBOSE=1 : Always show full JIT compilation logs (for debugging).\n" + + super().__init__(msg) def __new__(cls: type[Self], *args: Any, **kwargs: Any) -> Self: # noqa: ANN401, ARG004 if cls is JITCompileError: diff --git a/tests/test_config.py b/tests/test_config.py index a519ccd..05c2eea 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,7 @@ from __future__ import annotations +import multiprocessing +import os from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -10,6 +12,18 @@ import charonload +def _true_values() -> list[str]: + return ["1", "on", "On", "ON", "true", "TrUe", "True", "yes", "Yes", "YES", "y", "Y"] + + +def _false_values() -> list[str]: + return ["0", "off", "Off", "OFF", "false", "FaLsE", "False", "no", "No", "NO", "n", "N"] + + +def _error_values() -> list[str]: + return ["OOPS", "42", "f"] + + def test_provided_build_directory(shared_datadir: pathlib.Path, tmp_path: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" build_directory = tmp_path / "build" @@ -82,6 +96,89 @@ def test_relative_build_directory(shared_datadir: pathlib.Path) -> None: assert exc_info.type is ValueError +def _force_clean_build( + shared_datadir: pathlib.Path, + environ_value: str, + value: bool, # noqa: FBT001 + expected_value: bool, # noqa: FBT001 +) -> None: + os.environ["CHARONLOAD_FORCE_CLEAN_BUILD"] = environ_value + + project_directory = shared_datadir / "torch_cpu" + + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( + project_directory, + clean_build=value, + ) + config = module_config["test"] + + assert config.clean_build == expected_value + + +def _force_clean_build_error(shared_datadir: pathlib.Path, environ_value: str) -> None: + os.environ["CHARONLOAD_FORCE_CLEAN_BUILD"] = environ_value + + project_directory = shared_datadir / "torch_cpu" + + module_config = charonload.ConfigDict() + with pytest.raises(ValueError) as exc_info: + module_config["test"] = charonload.Config( + project_directory, + ) + + assert exc_info.type is ValueError + + +@pytest.mark.parametrize("environ_value", _true_values()) +def test_force_clean_build_true(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_clean_build, + args=( + shared_datadir, + environ_value, + False, + True, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + +@pytest.mark.parametrize("environ_value", _false_values()) +def test_force_clean_build_false(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_clean_build, + args=( + shared_datadir, + environ_value, + True, + False, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + +@pytest.mark.parametrize("environ_value", _error_values()) +def test_force_clean_build_error(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_clean_build_error, + args=( + shared_datadir, + environ_value, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + def test_prohibited_cmake_option_configuration_types(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" @@ -218,3 +315,169 @@ def test_relative_stubs_directory(shared_datadir: pathlib.Path) -> None: ) assert exc_info.type is ValueError + + +def _force_stubs_invalid_ok( + shared_datadir: pathlib.Path, + environ_value: str, + value: bool, # noqa: FBT001 + expected_value: bool, # noqa: FBT001 +) -> None: + os.environ["CHARONLOAD_FORCE_STUBS_INVALID_OK"] = environ_value + + project_directory = shared_datadir / "torch_cpu" + + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( + project_directory, + stubs_invalid_ok=value, + ) + config = module_config["test"] + + assert config.stubs_invalid_ok == expected_value + + +def _force_stubs_invalid_ok_error(shared_datadir: pathlib.Path, environ_value: str) -> None: + os.environ["CHARONLOAD_FORCE_STUBS_INVALID_OK"] = environ_value + + project_directory = shared_datadir / "torch_cpu" + + module_config = charonload.ConfigDict() + with pytest.raises(ValueError) as exc_info: + module_config["test"] = charonload.Config( + project_directory, + ) + + assert exc_info.type is ValueError + + +@pytest.mark.parametrize("environ_value", _true_values()) +def test_force_stubs_invalid_ok_true(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_stubs_invalid_ok, + args=( + shared_datadir, + environ_value, + False, + True, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + +@pytest.mark.parametrize("environ_value", _false_values()) +def test_force_stubs_invalid_ok_false(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_stubs_invalid_ok, + args=( + shared_datadir, + environ_value, + True, + False, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + +@pytest.mark.parametrize("environ_value", _error_values()) +def test_force_stubs_invalid_ok_error(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_stubs_invalid_ok_error, + args=( + shared_datadir, + environ_value, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + +def _force_verbose( + shared_datadir: pathlib.Path, + environ_value: str, + value: bool, # noqa: FBT001 + expected_value: bool, # noqa: FBT001 +) -> None: + os.environ["CHARONLOAD_FORCE_VERBOSE"] = environ_value + + project_directory = shared_datadir / "torch_cpu" + + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( + project_directory, + verbose=value, + ) + config = module_config["test"] + + assert config.verbose == expected_value + + +def _force_verbose_error(shared_datadir: pathlib.Path, environ_value: str) -> None: + os.environ["CHARONLOAD_FORCE_VERBOSE"] = environ_value + + project_directory = shared_datadir / "torch_cpu" + + module_config = charonload.ConfigDict() + with pytest.raises(ValueError) as exc_info: + module_config["test"] = charonload.Config( + project_directory, + ) + + assert exc_info.type is ValueError + + +@pytest.mark.parametrize("environ_value", _true_values()) +def test_force_verbose_true(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_verbose, + args=( + shared_datadir, + environ_value, + False, + True, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + +@pytest.mark.parametrize("environ_value", _false_values()) +def test_force_verbose_false(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_verbose, + args=( + shared_datadir, + environ_value, + True, + False, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0 + + +@pytest.mark.parametrize("environ_value", _error_values()) +def test_force_verbose_error(shared_datadir: pathlib.Path, environ_value: str) -> None: + p = multiprocessing.get_context("spawn").Process( + target=_force_verbose_error, + args=( + shared_datadir, + environ_value, + ), + ) + + p.start() + p.join() + assert p.exitcode == 0