diff --git a/docs/src/python/charonload.rst b/docs/src/python/charonload.rst index 48f063f..ff1ff7d 100644 --- a/docs/src/python/charonload.rst +++ b/docs/src/python/charonload.rst @@ -13,8 +13,10 @@ charonload/CMakeConfigureError charonload/CommandNotFoundError charonload/Config + charonload/ConfigDict charonload/JITCompileFinder charonload/JITCompileError + charonload/ResolvedConfig charonload/StubGenerationError charonload/extension_finder charonload/module_config diff --git a/docs/src/python/charonload/ConfigDict.rst b/docs/src/python/charonload/ConfigDict.rst new file mode 100644 index 0000000..7fb0d36 --- /dev/null +++ b/docs/src/python/charonload/ConfigDict.rst @@ -0,0 +1,7 @@ +ConfigDict +========== + +.. currentmodule:: charonload + +.. autoclass:: ConfigDict + :special-members: __setitem__ diff --git a/docs/src/python/charonload/ResolvedConfig.rst b/docs/src/python/charonload/ResolvedConfig.rst new file mode 100644 index 0000000..2e9e9a8 --- /dev/null +++ b/docs/src/python/charonload/ResolvedConfig.rst @@ -0,0 +1,6 @@ +ResolvedConfig +============== + +.. currentmodule:: charonload + +.. autoclass:: ResolvedConfig diff --git a/src/charonload/__init__.py b/src/charonload/__init__.py index 187e3c2..dc5942d 100644 --- a/src/charonload/__init__.py +++ b/src/charonload/__init__.py @@ -32,7 +32,7 @@ import email.utils import importlib.metadata -from ._config import Config +from ._config import Config, ConfigDict, ResolvedConfig from ._errors import ( BuildError, CMakeConfigureError, @@ -57,9 +57,11 @@ "CMakeConfigureError", "CommandNotFoundError", "Config", + "ConfigDict", "extension_finder", "JITCompileError", "JITCompileFinder", "module_config", + "ResolvedConfig", "StubGenerationError", ] diff --git a/src/charonload/_config.py b/src/charonload/_config.py index 5634674..ed438aa 100644 --- a/src/charonload/_config.py +++ b/src/charonload/_config.py @@ -9,6 +9,7 @@ import sys import sysconfig import tempfile +from collections import UserDict from dataclasses import dataclass from typing import TYPE_CHECKING @@ -20,9 +21,85 @@ colorama.just_fix_windows_console() -@dataclass(init=False) +@dataclass(init=False) # Python 3.10+: Use "_: KW_ONLY" class Config: - """The set of configuration options required for the import logic of the :class:`JITCompileFinder`.""" + """ + Set of user-specified configuration options required for the import logic of the :class:`JITCompileFinder`. + + This will be resolved into :class:`ResolvedConfig`. + """ + + project_directory: pathlib.Path | str + """The absolute path to the root directory of the C++/CUDA extension containing the root ``CMakeLists.txt`` file.""" + + build_directory: pathlib.Path | str | None + """ + An optional absolute path to a build directory. + + If not specified, the build will be placed in the temporary directory of the operating system. + """ + + clean_build: bool + """ + Whether to remove all cached files of previous builds from the build directory. + + This is useful to ensure consistent behavior after major changes in the CMake files of the project. + """ + + build_type: str + """The build type passed to CMake to compile the extension.""" + + cmake_options: dict[str, str] | None + """Additional CMake options to pass to the project when JIT compiling.""" + + stubs_directory: pathlib.Path | str | None + """ + An optional absolute path to the directory where stub files of the extension should be generated. + + This is useful for IDEs to get syntax highlighting and auto-completion for the extension content. For VS Code, the + respective (default) directory to specify here is ``/typings``. Stub generation is disabled + if set to ``None``. + """ + + stubs_invalid_ok: bool + """Whether to accept invalid stubs and skip raising an error.""" + + verbose: bool + """ + Whether to enable printing the full log of the JIT compilation. + + This is useful for debugging. + """ + + def __init__( + self: Self, + project_directory: pathlib.Path | str, + build_directory: pathlib.Path | str | None = None, + *, + clean_build: bool = False, + build_type: str = "RelWithDebInfo", + cmake_options: dict[str, str] | None = None, + stubs_directory: pathlib.Path | str | None = None, + stubs_invalid_ok: bool = False, + verbose: bool = False, + ) -> None: + self.project_directory = project_directory + self.build_directory = build_directory + self.clean_build = clean_build + self.build_type = build_type + self.cmake_options = cmake_options + self.stubs_directory = stubs_directory + self.stubs_invalid_ok = stubs_invalid_ok + self.verbose = verbose + + +@dataclass(init=False) # Python 3.10+: Use "kw_only=True" +class ResolvedConfig: + """ + Set of resolved configuration options that are actually used for the import logic of the :class:`JITCompileFinder`. + + This has been resolved from :class:`Config`. + """ full_project_directory: pathlib.Path """The full absolute path to the project directory.""" @@ -50,69 +127,76 @@ class Config: def __init__( self: Self, - project_directory: pathlib.Path | str, - build_directory: pathlib.Path | str | None = None, *, - clean_build: bool = False, - build_type: str = "RelWithDebInfo", - cmake_options: dict[str, str] | None = None, - stubs_directory: pathlib.Path | str | None = None, - stubs_invalid_ok: bool = False, - verbose: bool = False, + full_project_directory: pathlib.Path, + full_build_directory: pathlib.Path, + clean_build: bool, + build_type: str, + cmake_options: dict[str, str], + full_stubs_directory: pathlib.Path | None, + stubs_invalid_ok: bool, + verbose: bool, ) -> None: + self.full_project_directory = full_project_directory + self.full_build_directory = full_build_directory + self.clean_build = clean_build + self.build_type = build_type + self.cmake_options = cmake_options + self.full_stubs_directory = full_stubs_directory + self.stubs_invalid_ok = stubs_invalid_ok + self.verbose = verbose + + +class ConfigDict(UserDict[str, ResolvedConfig]): + """ + A configuration dictionary for holding resolved :class:`Config` instances. + + Configurations will be resolved during insertion into the dictionary. + """ + + def __setitem__(self: Self, key: str, value: Config | ResolvedConfig) -> None: """ - Create the configuration options from the provided parameters. + Resolve a user-specified configuration :class:`Config` into :class:`ResolvedConfig` and insert it. Parameters ---------- - project_directory - The absolute path to the root directory of the C++/CUDA extension containing the root ``CMakeLists.txt`` - file. - build_directory - An optional absolute path to a build directory. If not specified, the build will be placed in the - temporary directory of the operating system. - clean_build - Whether to remove all cached files of previous builds from the build directory. This is useful to ensure - consistent behavior after major changes in the CMake files of the project. - build_type - The build type passed to CMake to compile the extension. - cmake_options - Additional CMake options to pass to the project when JIT compiling. - stubs_directory - An optional absolute path to the directory where stub files of the extension should be generated. This is - useful for IDEs to get syntax highlighting and auto-completion for the extension content. For VS Code, the - respective (default) directory to specify here is ``/typings``. Stub generation is - disabled if set to ``None``. - stubs_invalid_ok - Whether to accept invalid stubs and skip raising an error. - verbose - Whether to enable printing the full log of the JIT compilation. This is useful for debugging. + key + The associated key of the configuration. + value + A user-specified or already resolved configuration. Raises ------ ValueError - If either: - 1) ``project_directory``,``build_directory``, or ``stubs_directory`` are not absolute paths, - 2) ``project_directory`` does not exists, or - 3) Prohibited options are inserted into ``cmake_options``. + During resolution if: + 1) ``config.project_directory``, ``config.build_directory``, or ``config.stubs_directory`` are not + absolute paths, + 2) ``config.project_directory`` does not exist, or + 3) ``config.cmake_options`` contains prohibited options. """ - if not pathlib.Path(project_directory).is_absolute(): - msg = f'Expected absolute project directory, but got relative directory "{project_directory}"' + super().__setitem__( + key, + self._resolve(value) if isinstance(value, Config) else value, + ) + + def _resolve(self: Self, config: Config) -> ResolvedConfig: + if not pathlib.Path(config.project_directory).is_absolute(): + msg = f'Expected absolute project directory, but got relative directory "{config.project_directory}"' raise ValueError(msg) - if not pathlib.Path(project_directory).resolve().exists(): - msg = f'Expected existing project directory, but got non-existing directory "{project_directory}"' + if not pathlib.Path(config.project_directory).resolve().exists(): + msg = f'Expected existing project directory, but got non-existing directory "{config.project_directory}"' raise ValueError(msg) - if build_directory is not None and not pathlib.Path(build_directory).is_absolute(): - msg = f'Expected absolute build directory, but got relative directory "{build_directory}"' + if config.build_directory is not None and not pathlib.Path(config.build_directory).is_absolute(): + msg = f'Expected absolute build directory, but got relative directory "{config.build_directory}"' raise ValueError(msg) - if stubs_directory is not None and not pathlib.Path(stubs_directory).is_absolute(): - msg = f'Expected absolute stub directory, but got relative directory "{stubs_directory}"' + if config.stubs_directory is not None and not pathlib.Path(config.stubs_directory).is_absolute(): + msg = f'Expected absolute stub directory, but got relative directory "{config.stubs_directory}"' raise ValueError(msg) - if cmake_options is not None: + if config.cmake_options is not None: prohibited_cmake_options = { "CHARONLOAD_.*", "CMAKE_CONFIGURATION_TYPES", @@ -121,27 +205,29 @@ def __init__( "TORCH_EXTENSION_NAME", } - for k in cmake_options: + for k in config.cmake_options: for pk in prohibited_cmake_options: if re.search(pk, k) is not None: msg = f'Found prohibited CMake option="{k}" which is not allowed or supported.' raise ValueError(msg) - self.full_project_directory = pathlib.Path(project_directory).resolve() - self.full_build_directory = self._find_build_directory( - build_directory=build_directory, - project_directory=project_directory, - verbose=verbose, + return ResolvedConfig( + full_project_directory=pathlib.Path(config.project_directory).resolve(), + full_build_directory=self._find_build_directory( + build_directory=config.build_directory, + project_directory=config.project_directory, + verbose=config.verbose, + ), + clean_build=config.clean_build, + build_type=config.build_type, + cmake_options=config.cmake_options if config.cmake_options is not None else {}, + full_stubs_directory=self._find_stubs_directory( + stubs_directory=config.stubs_directory, + verbose=config.verbose, + ), + stubs_invalid_ok=config.stubs_invalid_ok, + verbose=config.verbose, ) - self.clean_build = clean_build - self.build_type = build_type - self.cmake_options = cmake_options if cmake_options is not None else {} - self.full_stubs_directory = self._find_stubs_directory( - stubs_directory=stubs_directory, - verbose=verbose, - ) - self.stubs_invalid_ok = stubs_invalid_ok - self.verbose = verbose def _find_build_directory( self: Self, diff --git a/src/charonload/_finder.py b/src/charonload/_finder.py index 93af3b9..c13da57 100644 --- a/src/charonload/_finder.py +++ b/src/charonload/_finder.py @@ -23,7 +23,7 @@ from ._compat.typing import Self -from ._config import Config +from ._config import ConfigDict, ResolvedConfig from ._errors import BuildError, CMakeConfigureError, StubGenerationError from ._persistence import ( _EnumSerializer, @@ -94,8 +94,8 @@ def find_spec(self: Self, fullname, path, target=None) -> None: # noqa: ANN001, return None # noqa: PLR1711, RET501 -def _load(module_name: str, config: Config) -> None: - if not isinstance(config, Config): +def _load(module_name: str, config: ResolvedConfig) -> None: + if not isinstance(config, ResolvedConfig): msg = f"Invalid type of configuration: expected 'Config', but got '{config.__class__.__name__}'" raise TypeError(msg) @@ -124,7 +124,7 @@ class _JITCompileStep(ABC): exception_cls = type(None) step_name = "" - def __init__(self: Self, module_name: str, config: Config, step_number: tuple[int, int]) -> None: + def __init__(self: Self, module_name: str, config: ResolvedConfig, step_number: tuple[int, int]) -> None: self.module_name = module_name self.config = config self.step_number = step_number @@ -151,7 +151,7 @@ class _CleanStep(_JITCompileStep): exception_cls = type(None) step_name = "Clean" - def __init__(self: Self, module_name: str, config: Config, step_number: tuple[int, int]) -> None: + def __init__(self: Self, module_name: str, config: ResolvedConfig, step_number: tuple[int, int]) -> None: super().__init__(module_name, config, step_number) self.cache.connect( "status_cmake_configure", @@ -215,7 +215,7 @@ class _InitializeStep(_JITCompileStep): exception_cls = type(None) step_name = "Initialize" - def __init__(self: Self, module_name: str, config: Config, step_number: tuple[int, int]) -> None: + def __init__(self: Self, module_name: str, config: ResolvedConfig, step_number: tuple[int, int]) -> None: super().__init__(module_name, config, step_number) self.cache.connect( "version", @@ -234,7 +234,7 @@ class _CMakeConfigureStep(_JITCompileStep): exception_cls = CMakeConfigureError step_name = "CMake Configure" - def __init__(self: Self, module_name: str, config: Config, step_number: tuple[int, int]) -> None: + def __init__(self: Self, module_name: str, config: ResolvedConfig, step_number: tuple[int, int]) -> None: super().__init__(module_name, config, step_number) self.cache.connect( "status_cmake_configure", @@ -293,7 +293,7 @@ class _BuildStep(_JITCompileStep): exception_cls = BuildError step_name = "Build" - def __init__(self: Self, module_name: str, config: Config, step_number: tuple[int, int]) -> None: + def __init__(self: Self, module_name: str, config: ResolvedConfig, step_number: tuple[int, int]) -> None: super().__init__(module_name, config, step_number) self.cache.connect( "status_cmake_configure", @@ -334,7 +334,7 @@ class _StubGenerationStep(_JITCompileStep): exception_cls = StubGenerationError step_name = "Stub Generation" - def __init__(self: Self, module_name: str, config: Config, step_number: tuple[int, int]) -> None: + def __init__(self: Self, module_name: str, config: ResolvedConfig, step_number: tuple[int, int]) -> None: super().__init__(module_name, config, step_number) self.cache.connect( "status_stub_generation", @@ -412,7 +412,7 @@ class _ImportPathStep(_JITCompileStep): exception_cls = type(None) step_name = "Import Path" - def __init__(self: Self, module_name: str, config: Config, step_number: tuple[int, int]) -> None: + def __init__(self: Self, module_name: str, config: ResolvedConfig, step_number: tuple[int, int]) -> None: super().__init__(module_name, config, step_number) self.cache.connect( "location", @@ -450,11 +450,12 @@ def _run_impl(self: Self) -> None: ) -module_config: dict[str, Config] = {} +module_config: ConfigDict = ConfigDict() """ -The dictionary storing the registered :class:`Config ` instances. +The dictionary storing the registered :class:`ResolvedConfig ` instances. -Insert all C++/CUDA extension configurations into this dictionary to make the extensions importable. +Insert all C++/CUDA extension configurations via :class:`Config ` instances into this +dictionary to make the respective extensions importable. """ extension_finder: JITCompileFinder = JITCompileFinder() diff --git a/tests/test_config.py b/tests/test_config.py index ee755a8..a519ccd 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,10 +15,12 @@ def test_provided_build_directory(shared_datadir: pathlib.Path, tmp_path: pathli build_directory = tmp_path / "build" assert not build_directory.exists() - config = charonload.Config( + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( project_directory, build_directory, ) + config = module_config["test"] config.full_build_directory.mkdir(parents=True) @@ -30,9 +32,11 @@ def test_provided_build_directory(shared_datadir: pathlib.Path, tmp_path: pathli def test_fallback_build_directory(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" - config = charonload.Config( + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( project_directory, ) + config = module_config["test"] config.full_build_directory.mkdir(parents=True, exist_ok=True) @@ -43,8 +47,9 @@ def test_fallback_build_directory(shared_datadir: pathlib.Path) -> None: def test_relative_project_directory() -> None: project_directory = "some/relative/path" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, ) @@ -54,8 +59,9 @@ def test_relative_project_directory() -> None: def test_non_existing_project_directory(tmp_path: pathlib.Path) -> None: project_directory = tmp_path / "some/absolute/non-existing/path" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, ) @@ -66,8 +72,9 @@ def test_relative_build_directory(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" build_directory = "some/relative/path" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, build_directory, ) @@ -78,8 +85,9 @@ def test_relative_build_directory(shared_datadir: pathlib.Path) -> None: def test_prohibited_cmake_option_configuration_types(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, cmake_options={"CMAKE_CONFIGURATION_TYPES": "Release"}, ) @@ -90,8 +98,9 @@ def test_prohibited_cmake_option_configuration_types(shared_datadir: pathlib.Pat def test_prohibited_cmake_option_prefix_path(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, cmake_options={"CMAKE_PREFIX_PATH": "some/path"}, ) @@ -102,8 +111,9 @@ def test_prohibited_cmake_option_prefix_path(shared_datadir: pathlib.Path) -> No def test_prohibited_cmake_option_project_top_level_includes(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, cmake_options={"CMAKE_PROJECT_TOP_LEVEL_INCLUDES": "some/path"}, ) @@ -114,8 +124,9 @@ def test_prohibited_cmake_option_project_top_level_includes(shared_datadir: path def test_prohibited_cmake_option_torch_extension_name(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, cmake_options={"TORCH_EXTENSION_NAME": "Name"}, ) @@ -126,8 +137,9 @@ def test_prohibited_cmake_option_torch_extension_name(shared_datadir: pathlib.Pa def test_prohibited_cmake_option_reserved_symbols(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, cmake_options={"CHARONLOAD_TEST": "some value"}, ) @@ -138,10 +150,12 @@ def test_prohibited_cmake_option_reserved_symbols(shared_datadir: pathlib.Path) def test_none_cmake_options(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" - config = charonload.Config( + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( project_directory, cmake_options=None, ) + config = module_config["test"] assert len(config.cmake_options) == 0 @@ -149,10 +163,12 @@ def test_none_cmake_options(shared_datadir: pathlib.Path) -> None: def test_empty_cmake_options(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" - config = charonload.Config( + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( project_directory, cmake_options={}, ) + config = module_config["test"] assert len(config.cmake_options) == 0 @@ -162,10 +178,12 @@ def test_stubs_directory(shared_datadir: pathlib.Path, tmp_path: pathlib.Path) - stubs_directory = tmp_path / "typings" assert not stubs_directory.exists() - config = charonload.Config( + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( project_directory, stubs_directory=stubs_directory, ) + config = module_config["test"] assert config.full_stubs_directory is not None config.full_stubs_directory.mkdir(parents=True) @@ -178,10 +196,12 @@ def test_stubs_directory(shared_datadir: pathlib.Path, tmp_path: pathlib.Path) - def test_no_stubs_generation(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" - config = charonload.Config( + module_config = charonload.ConfigDict() + module_config["test"] = charonload.Config( project_directory, stubs_directory=None, ) + config = module_config["test"] assert config.full_stubs_directory is None @@ -190,8 +210,9 @@ def test_relative_stubs_directory(shared_datadir: pathlib.Path) -> None: project_directory = shared_datadir / "torch_cpu" stubs_directory = "some/relative/path" + module_config = charonload.ConfigDict() with pytest.raises(ValueError) as exc_info: - charonload.Config( + module_config["test"] = charonload.Config( project_directory, stubs_directory=stubs_directory, )