diff --git a/PIXI_PLAN.md b/PIXI_PLAN.md new file mode 100644 index 00000000..dadeceae --- /dev/null +++ b/PIXI_PLAN.md @@ -0,0 +1,111 @@ +# Pixi Integration - Simple Implementation Plan + +## Core Philosophy +**Let UniDep translate, let Pixi resolve** + +UniDep should act as a simple translator from `requirements.yaml`/`pyproject.toml` to `pixi.toml` format. +Pixi handles all dependency resolution, conflict management, and lock file generation. + +## Current Problem with `pixi` Branch +- **Over-engineering**: Pre-resolves conflicts that Pixi can handle +- **Origin tracking**: Complex system to track where each dependency came from +- **Unnecessary complexity**: ~500+ lines of code for what should be ~100 lines + +## New Simple Architecture + +### Phase 1: Basic Pixi.toml Generation ✅ +- [x] Create minimal `_pixi.py` module (~100 lines) +- [ ] Parse requirements WITHOUT resolution +- [ ] Create features with literal dependencies +- [ ] Generate pixi.toml with proper structure +- [ ] Add `--pixi` flag to merge command + +### Phase 2: Pixi Lock Command +- [ ] Add `pixi-lock` subcommand to CLI +- [ ] Simple wrapper around `pixi lock` command +- [ ] Support platform selection +- [ ] Add basic tests + +### Phase 3: Monorepo Support (Optional) +- [ ] Generate sub-lock files if needed +- [ ] But let Pixi handle the complexity + +## Implementation Details + +### 1. Simple Pixi.toml Structure +```python +def generate_pixi_toml(requirements_files, output_file): + pixi_data = { + "project": { + "name": "myenv", + "channels": channels, + "platforms": platforms, + }, + "dependencies": {}, + "pypi-dependencies": {}, + } + + # For monorepo: create features + if len(requirements_files) > 1: + pixi_data["feature"] = {} + pixi_data["environments"] = {} + + for req_file in requirements_files: + feature_name = req_file.parent.stem + deps = parse_single_file(req_file) # NO RESOLUTION! + + pixi_data["feature"][feature_name] = { + "dependencies": deps.conda, # Literal copy + "pypi-dependencies": deps.pip, # Literal copy + } + + # Create environments + all_features = list(pixi_data["feature"].keys()) + pixi_data["environments"]["default"] = all_features + for feat in all_features: + pixi_data["environments"][feat.replace("_", "-")] = [feat] +``` + +### 2. Key Simplifications +- **NO conflict resolution** - Pixi handles this +- **NO origin tracking** - Not needed +- **NO version pinning combination** - Pixi does this +- **NO platform resolution** - Use Pixi's native platform support + +### 3. Testing Strategy +- Test pixi.toml generation (structure validation) +- Test CLI integration +- Test with example monorepo +- Let Pixi handle the actual resolution testing + +## Files to Create/Modify + +### New Files +1. `unidep/_pixi.py` - Simple pixi.toml generation (~100 lines) +2. `tests/test_pixi.py` - Basic tests (~50 lines) + +### Modified Files +1. `unidep/_cli.py` - Add `--pixi` flag and `pixi-lock` command +2. `README.md` - Document new Pixi support + +## Success Criteria +- [ ] Generate valid pixi.toml files +- [ ] Pass all tests +- [ ] Work with monorepo example +- [ ] Total implementation < 200 lines (vs 500+ in old branch) + +## Timeline +- **Hour 1-2**: Basic pixi.toml generation ✅ +- **Hour 3-4**: CLI integration and testing +- **Hour 5-6**: Documentation and polish + +## Testing Checkpoints +After each major change: +1. Run tests: `pytest tests/test_pixi.py -xvs` +2. Test with monorepo: `unidep merge --pixi tests/simple_monorepo` +3. Validate pixi.toml: `pixi list` + +## Commit Strategy +- Commit after each working phase +- Clear commit messages +- Test before each commit diff --git a/README.md b/README.md index 73024374..140d087d 100644 --- a/README.md +++ b/README.md @@ -550,7 +550,8 @@ positional arguments: Subcommands merge Combine multiple (or a single) `requirements.yaml` or `pyproject.toml` files into a single Conda installable - `environment.yaml` file. + `environment.yaml` file or Pixi installable + `pixi.toml` file. install Automatically install all dependencies from one or more `requirements.yaml` or `pyproject.toml` files. This command first installs dependencies with Conda, @@ -603,23 +604,24 @@ See `unidep merge -h` for more information: ```bash usage: unidep merge [-h] [-o OUTPUT] [-n NAME] [--stdout] - [--selector {sel,comment}] [-d DIRECTORY] [--depth DEPTH] - [-v] + [--selector {sel,comment}] [--pixi] [-d DIRECTORY] + [--depth DEPTH] [-v] [-p {linux-64,linux-aarch64,linux-ppc64le,osx-64,osx-arm64,win-64}] [--skip-dependency SKIP_DEPENDENCY] [--ignore-pin IGNORE_PIN] [--overwrite-pin OVERWRITE_PIN] Combine multiple (or a single) `requirements.yaml` or `pyproject.toml` files -into a single Conda installable `environment.yaml` file. Example usage: -`unidep merge --directory . --depth 1 --output environment.yaml` to search for -`requirements.yaml` or `pyproject.toml` files in the current directory and its -subdirectories and create `environment.yaml`. These are the defaults, so you -can also just run `unidep merge`. +into a single Conda installable `environment.yaml` file or Pixi installable +`pixi.toml` file. Example usage: `unidep merge --directory . --depth 1 +--output environment.yaml` to search for `requirements.yaml` or +`pyproject.toml` files in the current directory and its subdirectories and +create `environment.yaml`. These are the defaults, so you can also just run +`unidep merge`. options: -h, --help show this help message and exit -o, --output OUTPUT Output file for the conda environment, by default - `environment.yaml` + `environment.yaml` or `pixi.toml` if `--pixi` is used -n, --name NAME Name of the conda environment, by default `myenv` --stdout Output to stdout instead of a file --selector {sel,comment} @@ -627,6 +629,8 @@ options: `sel` then `- numpy # [linux]` becomes `sel(linux): numpy`, if `comment` then it remains `- numpy # [linux]`, by default `sel` + --pixi Generate a `pixi.toml` file instead of + `environment.yaml` -d, --directory DIRECTORY Base directory to scan for `requirements.yaml` or `pyproject.toml` file(s), by default `.` diff --git a/tests/test_pixi.py b/tests/test_pixi.py new file mode 100644 index 00000000..52bea7a6 --- /dev/null +++ b/tests/test_pixi.py @@ -0,0 +1,242 @@ +"""Tests for simple Pixi.toml generation.""" + +from __future__ import annotations + +import textwrap +from typing import TYPE_CHECKING + +from unidep._pixi import generate_pixi_toml + +if TYPE_CHECKING: + from pathlib import Path + + +def test_simple_pixi_generation(tmp_path: Path) -> None: + """Test basic pixi.toml generation from a single requirements.yaml.""" + req_file = tmp_path / "requirements.yaml" + req_file.write_text( + textwrap.dedent( + """\ + channels: + - conda-forge + dependencies: + - numpy >=1.20 + - pandas + - pip: requests + platforms: + - linux-64 + - osx-arm64 + """, + ), + ) + + output_file = tmp_path / "pixi.toml" + generate_pixi_toml( + req_file, + project_name="test-project", + output_file=output_file, + verbose=False, + ) + + assert output_file.exists() + content = output_file.read_text() + + # Check basic structure + assert "[project]" in content + assert 'name = "test-project"' in content + assert "conda-forge" in content + assert "linux-64" in content + assert "osx-arm64" in content + + # Check dependencies + assert "[dependencies]" in content + assert 'numpy = ">=1.20"' in content + assert 'pandas = "*"' in content + + assert "[pypi-dependencies]" in content + assert 'requests = "*"' in content + + +def test_monorepo_pixi_generation(tmp_path: Path) -> None: + """Test pixi.toml generation with features for multiple requirements files.""" + # Create project1 + project1_dir = tmp_path / "project1" + project1_dir.mkdir() + req1 = project1_dir / "requirements.yaml" + req1.write_text( + textwrap.dedent( + """\ + channels: + - conda-forge + dependencies: + - numpy + - conda: scipy + """, + ), + ) + + # Create project2 + project2_dir = tmp_path / "project2" + project2_dir.mkdir() + req2 = project2_dir / "requirements.yaml" + req2.write_text( + textwrap.dedent( + """\ + channels: + - conda-forge + dependencies: + - pandas + - pip: requests + """, + ), + ) + + output_file = tmp_path / "pixi.toml" + generate_pixi_toml( + req1, + req2, + project_name="monorepo", + output_file=output_file, + verbose=False, + ) + + assert output_file.exists() + content = output_file.read_text() + + # Check project section + assert "[project]" in content + assert 'name = "monorepo"' in content + + # Check feature dependencies (TOML writes them directly without parent section) + assert "[feature.project1.dependencies]" in content + assert 'numpy = "*"' in content + assert 'scipy = "*"' in content + + assert "[feature.project2.dependencies]" in content + assert 'pandas = "*"' in content + + assert "[feature.project2.pypi-dependencies]" in content + assert 'requests = "*"' in content + + # Check environments (be flexible with TOML formatting) + assert "[environments]" in content + assert "default =" in content + assert "project1" in content + assert "project2" in content + # Verify that default includes both projects + assert content.count('"project1"') >= 2 # In default and individual env + assert content.count('"project2"') >= 2 # In default and individual env + + +def test_pixi_with_version_pins(tmp_path: Path) -> None: + """Test that version pins are passed through without resolution.""" + req_file = tmp_path / "requirements.yaml" + req_file.write_text( + textwrap.dedent( + """\ + channels: + - conda-forge + dependencies: + - numpy >=1.20,<2.0 + - conda: scipy =1.9.0 + - pip: requests >2.20 + - sympy >= 1.11 + """, + ), + ) + + output_file = tmp_path / "pixi.toml" + generate_pixi_toml( + req_file, + output_file=output_file, + verbose=False, + ) + + content = output_file.read_text() + + # Check that pins are preserved exactly (spaces removed) + assert 'numpy = ">=1.20,<2.0"' in content + assert 'scipy = "=1.9.0"' in content + assert 'requests = ">2.20"' in content + assert 'sympy = ">=1.11"' in content # Space should be removed + + +def test_pixi_with_local_package(tmp_path: Path) -> None: + """Test that local packages are added as editable dependencies.""" + # Create a directory with requirements.yaml and pyproject.toml + project_dir = tmp_path / "my_package" + project_dir.mkdir() + + req_file = project_dir / "requirements.yaml" + req_file.write_text( + textwrap.dedent( + """\ + channels: + - conda-forge + dependencies: + - numpy + """, + ), + ) + + # Create a pyproject.toml with build-system to simulate a local package + pyproject_file = project_dir / "pyproject.toml" + pyproject_file.write_text( + textwrap.dedent( + """\ + [build-system] + requires = ["setuptools"] + + [project] + name = "my-package" + """, + ), + ) + + output_file = tmp_path / "pixi.toml" + generate_pixi_toml( + project_dir, + output_file=output_file, + verbose=False, + ) + + assert output_file.exists() + content = output_file.read_text() + + # Check that the local package is added as an editable dependency + # TOML can format this as either inline or table format + assert "pypi-dependencies" in content + assert "my_package" in content + assert 'path = "."' in content + assert "editable = true" in content + assert 'numpy = "*"' in content + + +def test_pixi_empty_dependencies(tmp_path: Path) -> None: + """Test handling of requirements file with no dependencies.""" + req_file = tmp_path / "requirements.yaml" + req_file.write_text( + textwrap.dedent( + """\ + channels: + - conda-forge + platforms: + - linux-64 + """, + ), + ) + + output_file = tmp_path / "pixi.toml" + generate_pixi_toml( + req_file, + output_file=output_file, + verbose=False, + ) + + assert output_file.exists() + content = output_file.read_text() + + # Should have project section but no dependencies sections + assert "[project]" in content + assert "[dependencies]" not in content + assert "[pypi-dependencies]" not in content diff --git a/unidep/_cli.py b/unidep/_cli.py index e820d6e1..3eecebd7 100755 --- a/unidep/_cli.py +++ b/unidep/_cli.py @@ -286,7 +286,8 @@ def _parse_args() -> argparse.Namespace: merge_help = ( f"Combine multiple (or a single) {_DEP_FILES}" " files into a" - " single Conda installable `environment.yaml` file." + " single Conda installable `environment.yaml` file" + " or Pixi installable `pixi.toml` file." ) merge_example = ( " Example usage: `unidep merge --directory . --depth 1 --output environment.yaml`" # noqa: E501 @@ -305,8 +306,9 @@ def _parse_args() -> argparse.Namespace: "-o", "--output", type=Path, - default="environment.yaml", - help="Output file for the conda environment, by default `environment.yaml`", + default=None, + help="Output file for the conda environment, by default `environment.yaml`" + " or `pixi.toml` if `--pixi` is used", ) parser_merge.add_argument( "-n", @@ -329,6 +331,11 @@ def _parse_args() -> argparse.Namespace: " `- numpy # [linux]` becomes `sel(linux): numpy`, if `comment` then" " it remains `- numpy # [linux]`, by default `sel`", ) + parser_merge.add_argument( + "--pixi", + action="store_true", + help="Generate a `pixi.toml` file instead of `environment.yaml`", + ) _add_common_args( parser_merge, { @@ -1268,17 +1275,20 @@ def _merge_command( directory: Path, files: list[Path] | None, name: str, - output: Path, + output: Path | None, stdout: bool, selector: Literal["sel", "comment"], platforms: list[Platform], ignore_pins: list[str], skip_dependencies: list[str], overwrite_pins: list[str], + pixi: bool, verbose: bool, ) -> None: # pragma: no cover # When using stdout, suppress verbose output verbose = verbose and not stdout + if output is None: + output = Path("pixi.toml") if pixi else Path("environment.yaml") if files: # ignores depth and directory! found_files = files @@ -1292,33 +1302,56 @@ def _merge_command( print(f"❌ No {_DEP_FILES} files found in {directory}") sys.exit(1) - requirements = parse_requirements( - *found_files, - ignore_pins=ignore_pins, - overwrite_pins=overwrite_pins, - skip_dependencies=skip_dependencies, - verbose=verbose, - ) - if not platforms: - platforms = requirements.platforms or [identify_current_platform()] - resolved = resolve_conflicts( - requirements.requirements, - platforms, - optional_dependencies=requirements.optional_dependencies, - ) - env_spec = create_conda_env_specification( - resolved, - requirements.channels, - platforms, - selector=selector, - ) - output_file = None if stdout else output - write_conda_environment_file(env_spec, output_file, name, verbose=verbose) - if output_file: - found_files_str = ", ".join(f"`{f}`" for f in found_files) - print( - f"✅ Generated environment file at `{output_file}` from {found_files_str}", + if pixi: + # Use the new simple Pixi generation + from unidep._pixi import generate_pixi_toml + + requirements = parse_requirements( + *found_files, + ignore_pins=ignore_pins, + overwrite_pins=overwrite_pins, + skip_dependencies=skip_dependencies, + verbose=verbose, ) + output_file = None if stdout else output + generate_pixi_toml( + *found_files, + project_name=name, + channels=requirements.channels, + platforms=requirements.platforms or platforms, + output_file=output_file, + verbose=verbose, + ) + else: + # Original conda environment generation + requirements = parse_requirements( + *found_files, + ignore_pins=ignore_pins, + overwrite_pins=overwrite_pins, + skip_dependencies=skip_dependencies, + verbose=verbose, + ) + if not platforms: + platforms = requirements.platforms or [identify_current_platform()] + resolved = resolve_conflicts( + requirements.requirements, + platforms, + optional_dependencies=requirements.optional_dependencies, + ) + env_spec = create_conda_env_specification( + resolved, + requirements.channels, + platforms, + selector=selector, + ) + output_file = None if stdout else output + write_conda_environment_file(env_spec, output_file, name, verbose=verbose) + if output_file: + found_files_str = ", ".join(f"`{f}`" for f in found_files) + print( + f"✅ Generated environment file at `{output_file}` " + f"from {found_files_str}", + ) def _pip_compile_command( @@ -1497,6 +1530,7 @@ def main() -> None: ignore_pins=args.ignore_pin, skip_dependencies=args.skip_dependency, overwrite_pins=args.overwrite_pin, + pixi=args.pixi, verbose=args.verbose, ) elif args.command == "pip": # pragma: no cover diff --git a/unidep/_conda_lock.py b/unidep/_conda_lock.py index 07c52a84..5a192aa8 100644 --- a/unidep/_conda_lock.py +++ b/unidep/_conda_lock.py @@ -119,6 +119,7 @@ def _conda_lock_global( overwrite_pins=overwrite_pins, skip_dependencies=skip_dependencies, verbose=verbose, + pixi=False, ) _run_conda_lock( tmp_env, diff --git a/unidep/_pixi.py b/unidep/_pixi.py new file mode 100644 index 00000000..10027861 --- /dev/null +++ b/unidep/_pixi.py @@ -0,0 +1,231 @@ +"""Simple Pixi.toml generation without conflict resolution.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from unidep._dependencies_parsing import parse_requirements +from unidep.utils import identify_current_platform, is_pip_installable + +if TYPE_CHECKING: + from unidep._dependencies_parsing import ParsedRequirements + from unidep.platform_definitions import Platform + +try: + if sys.version_info >= (3, 11): + import tomllib + else: + import tomli as tomllib + HAS_TOML = True +except ImportError: + HAS_TOML = False + + +def _get_package_name(project_dir: Path) -> str | None: + """Get the package name from pyproject.toml or setup.py.""" + pyproject_path = project_dir / "pyproject.toml" + if pyproject_path.exists() and HAS_TOML: + try: + with pyproject_path.open("rb") as f: + data = tomllib.load(f) + if "project" in data and "name" in data["project"]: + # Normalize package name for use in dependencies + # Replace dots and hyphens with underscores + name = data["project"]["name"] + return name.replace("-", "_").replace(".", "_") + except Exception: # noqa: S110, BLE001 + pass + # Fallback to directory name + return project_dir.name + + +def generate_pixi_toml( # noqa: PLR0912, C901, PLR0915 + *requirements_files: Path, + project_name: str | None = None, + channels: list[str] | None = None, + platforms: list[Platform] | None = None, + output_file: str | Path | None = "pixi.toml", + verbose: bool = False, +) -> None: + """Generate a pixi.toml file from requirements files. + + This function creates a pixi.toml with features for each requirements file, + letting Pixi handle all dependency resolution and conflict management. + """ + if not requirements_files: + requirements_files = (Path.cwd(),) + + # Initialize pixi structure + pixi_data: dict[str, Any] = {} + + # Collect channels and platforms from all requirements files + all_channels = set() + all_platforms = set() + + # If single file, put dependencies at root level + if len(requirements_files) == 1: + req = parse_requirements(requirements_files[0], verbose=verbose) + conda_deps, pip_deps = _extract_dependencies(req) + + # Use channels and platforms from the requirements file + if req.channels: + all_channels.update(req.channels) + if req.platforms: + all_platforms.update(req.platforms) + + if conda_deps: + pixi_data["dependencies"] = conda_deps + if pip_deps: + pixi_data["pypi-dependencies"] = pip_deps + + # Check if there's a local package in the same directory + req_file = requirements_files[0] + req_dir = req_file.parent if req_file.is_file() else req_file + if is_pip_installable(req_dir): + # Add the local package as an editable dependency + if "pypi-dependencies" not in pixi_data: + pixi_data["pypi-dependencies"] = {} + # Get the actual package name from pyproject.toml + package_name = _get_package_name(req_dir) or req_dir.name + pixi_data["pypi-dependencies"][package_name] = { + "path": ".", + "editable": True, + } + else: + # Multiple files: create features + pixi_data["feature"] = {} + pixi_data["environments"] = {} + all_features = [] + + for req_file in requirements_files: + feature_name = req_file.parent.stem if req_file.is_file() else req_file.stem + req = parse_requirements(req_file, verbose=verbose) + conda_deps, pip_deps = _extract_dependencies(req) + + # Collect channels and platforms + if req.channels: + all_channels.update(req.channels) + if req.platforms: + all_platforms.update(req.platforms) + + feature: dict[str, Any] = {} + if conda_deps: + feature["dependencies"] = conda_deps + if pip_deps: + feature["pypi-dependencies"] = pip_deps + + # Check if there's a local package in the same directory + req_dir = req_file.parent if req_file.is_file() else req_file + if is_pip_installable(req_dir): + # Add the local package as an editable dependency + if "pypi-dependencies" not in feature: + feature["pypi-dependencies"] = {} + # Get the actual package name from pyproject.toml + package_name = _get_package_name(req_dir) or feature_name + # Use relative path from the output file location + rel_path = f"./{feature_name}" + feature["pypi-dependencies"][package_name] = { + "path": rel_path, + "editable": True, + } + + if feature: # Only add non-empty features + pixi_data["feature"][feature_name] = feature + all_features.append(feature_name) + + # Create environments + if all_features: + pixi_data["environments"]["default"] = all_features + for feat in all_features: + # Environment names can't have underscores + env_name = feat.replace("_", "-") + pixi_data["environments"][env_name] = [feat] + + # Set project metadata with collected channels and platforms + pixi_data["project"] = { + "name": project_name or Path.cwd().name, + "channels": ( + list(all_channels) if all_channels else (channels or ["conda-forge"]) + ), + "platforms": ( + list(all_platforms) + if all_platforms + else (platforms or [identify_current_platform()]) + ), + } + + # Write the pixi.toml file + _write_pixi_toml(pixi_data, output_file, verbose=verbose) + + +def _extract_dependencies( + requirements: ParsedRequirements, +) -> tuple[dict[str, str], dict[str, str]]: + """Extract conda and pip dependencies from parsed requirements. + + Returns a tuple of (conda_deps, pip_deps) as simple name->version dicts. + No conflict resolution - just pass through what's specified. + """ + conda_deps = {} + pip_deps = {} + + # Process each package's specifications + for pkg_name, specs in requirements.requirements.items(): + conda_spec = None + pip_spec = None + + for spec in specs: + # Format the version pin or use "*" if no pin + version = spec.pin.replace(" ", "") if spec.pin else "*" + + # Add platform selector if present + if spec.selector: + # In pixi.toml, platform selectors go in target section + # For now, we'll skip platform-specific deps for simplicity + # This can be enhanced later if needed + continue + + if spec.which == "conda": + # Keep the conda spec, prefer pinned versions + if conda_spec is None or spec.pin: + conda_spec = version + elif spec.which == "pip" and (pip_spec is None or spec.pin): + # Keep the pip spec, prefer pinned versions + pip_spec = version + + # Add to appropriate section + if conda_spec: + conda_deps[pkg_name] = conda_spec + if pip_spec and pkg_name not in conda_deps: # Only add to pip if not in conda + pip_deps[pkg_name] = pip_spec + + return conda_deps, pip_deps + + +def _write_pixi_toml( + pixi_data: dict[str, Any], + output_file: str | Path | None, + *, + verbose: bool = False, +) -> None: + """Write the pixi data structure to a TOML file.""" + try: + import tomli_w + except ImportError: + msg = ( + "❌ `tomli_w` is required to write TOML files. " + "Install it with `pip install tomli_w`." + ) + raise ImportError(msg) from None + + if output_file is not None: + output_path = Path(output_file) + with output_path.open("wb") as f: + tomli_w.dump(pixi_data, f) + if verbose: + print(f"✅ Generated pixi.toml at {output_path}") + else: + # Output to stdout + tomli_w.dump(pixi_data, sys.stdout.buffer)