diff --git a/README.md b/README.md index 7633531..0cdb718 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,12 @@ pipeline so users can see the effect immediately. pip install Leakly ``` +For notebook environments that need the optional notebook dependencies: + +```bash +pip install "Leakly[notebook]" +``` + For the current GitHub checkout: ```bash @@ -88,9 +94,7 @@ After label permutation, there should be no real biological, clinical, or statis A leaky pipeline may still score well if information from the full dataset enters the analysis before the train/test split or outside the cross-validation loop. Common sources include feature selection, scaling, imputation, covariate adjustment, dimensionality reduction, or hyperparameter tuning performed on all samples. -This is especially problematic in high-dimensional data, such as neuroimaging, omics, or biomarker studies, where random label-specific patterns can appear meaningful by chance. If the test set influences preprocessing or feature selection, the model may preserve these random patterns and show inflated performance. - -Leakly tests this directly: does the pipeline still perform above chance when labels are meaningless? If yes, it does not pinpoint the exact leakage source, but it strongly indicates that the pipeline needs inspection. +This is especially problematic in high-dimensional data, such as neuroimaging, omics, or biomarker studies, where random label-specific patterns can appear meaningful by chance. If the test set influences preprocessing or feature selection, the model may "remember" these random patterns and show inflated performance. **How many permutations should I run?** diff --git a/example.ipynb b/example.ipynb index 20fc6dc..7980b94 100644 --- a/example.ipynb +++ b/example.ipynb @@ -30,20 +30,22 @@ "metadata": {}, "outputs": [], "source": [ - "# Install Leakly locally, or from GitHub when running in Colab.\n", + "# Install Leakly from this checkout locally, or from PyPI elsewhere.\n", "import importlib.metadata as metadata\n", + "from pathlib import Path\n", "import subprocess\n", "import sys\n", "\n", - "install_args = [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\"]\n", - "try:\n", - " import google.colab # type: ignore # noqa: F401\n", - " install_target = \"git+https://github.com/DeMONLab-BioFINDER/Leakly.git@main\"\n", - " install_args.append(\"--force-reinstall\")\n", - "except ImportError:\n", - " install_target = \".[notebook]\"\n", + "repo_root = Path.cwd()\n", + "local_checkout = (repo_root / \"pyproject.toml\").exists() and (repo_root / \"leakly\").is_dir()\n", "\n", - "subprocess.check_call([*install_args, install_target])\n", + "install_args = [sys.executable, \"-m\", \"pip\", \"install\", \"-q\"]\n", + "if local_checkout:\n", + " install_args.extend([\"-e\", \".[notebook]\"])\n", + "else:\n", + " install_args.extend([\"--upgrade\", \"Leakly\"])\n", + "\n", + "subprocess.check_call(install_args)\n", "\n", "for module_name in list(sys.modules):\n", " if module_name == \"leakly\" or module_name.startswith(\"leakly.\"):\n", @@ -169,10 +171,10 @@ "from leakly import permute_label\n", "from tqdm.auto import tqdm\n", "\n", - "# Use 2000 permutations for now to replicate `assets/AUC.png`.\n", - "# Use 100 permutations for a quick test run.\n", + "# Use 100 permutations for a quick example run.\n", + "# Increase to 2000 if you want to replicate `assets/AUC.png` more closely.\n", "# Adjust based on your computational resources and needs.\n", - "N_PERMUTATIONS = 2000\n", + "N_PERMUTATIONS = 100\n", "\n", "leakage_permuted_scores = []\n", "nonleakage_permuted_scores = []\n", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1c378f3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "Leakly" +dynamic = ["version"] +description = "Leakage checks for machine-learning pipelines using permutation tests." +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +license-files = ["LICENSE"] +authors = [ + { name = "DeMONLab-BioFINDER" }, +] +keywords = [ + "data-leakage", + "machine-learning", + "permutation-test", + "scikit-learn", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [ + "joblib>=1.3,<2.0", + "matplotlib>=3.7,<4.0", + "numpy>=1.23,<2.4", + "pandas>=1.5,<3.0", + "PyYAML>=6.0,<7.0", + "scikit-learn>=1.6,<1.7", + "scipy>=1.9,<1.16", + "tqdm>=4.64,<5.0", +] + +[project.optional-dependencies] +dev = [ + "build>=1.0", + "pytest>=8.0", + "pytest-cov>=5.0", + "twine>=5.0", + "wheel>=0.41", +] +notebook = [ + "ipykernel>=6.0", + "ipywidgets>=8.0", + "jupyter>=1.0", +] + +[project.urls] +Homepage = "https://github.com/DeMONLab-BioFINDER/Leakly" +Source = "https://github.com/DeMONLab-BioFINDER/Leakly" +Issues = "https://github.com/DeMONLab-BioFINDER/Leakly/issues" + +[tool.setuptools.dynamic] +version = { attr = "leakly.__version__" } + +[tool.setuptools.packages.find] +include = ["leakly", "leakly.*"] + +[tool.setuptools.package-data] +leakly = ["*.yaml"] + +[tool.setuptools] +include-package-data = true diff --git a/setup.py b/setup.py deleted file mode 100644 index d9e7f79..0000000 --- a/setup.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -"""Setuptools configuration for publishing Leakly on PyPI.""" -from pathlib import Path -import re - -from setuptools import find_packages, setup - - -ROOT = Path(__file__).parent - - -def read_readme() -> str: - return (ROOT / "README.md").read_text(encoding="utf-8") - - -def read_version() -> str: - init_file = (ROOT / "leakly" / "__init__.py").read_text(encoding="utf-8") - match = re.search(r'^__version__ = ["\']([^"\']+)["\']', init_file, re.M) - if not match: - raise RuntimeError("Unable to find __version__ in leakly/__init__.py") - return match.group(1) - - -setup( - name="Leakly", - version=read_version(), - description=( - "Leakage checks for machine-learning pipelines using permutation tests." - ), - long_description=read_readme(), - long_description_content_type="text/markdown", - author="DeMONLab-BioFINDER", - license="MIT", - url="https://github.com/DeMONLab-BioFINDER/Leakly", - project_urls={ - "Source": "https://github.com/DeMONLab-BioFINDER/Leakly", - "Issues": "https://github.com/DeMONLab-BioFINDER/Leakly/issues", - }, - packages=find_packages(include=["leakly", "leakly.*"]), - include_package_data=True, - package_data={"leakly": ["*.yaml"]}, - python_requires=">=3.10", - install_requires=[ - "joblib>=1.3,<2.0", - "matplotlib>=3.7,<4.0", - "numpy>=1.23,<2.4", - "pandas>=1.5,<3.0", - "PyYAML>=6.0,<7.0", - "scikit-learn>=1.6,<1.7", - "scipy>=1.9,<1.16", - "tqdm>=4.64,<5.0", - ], - extras_require={ - "dev": [ - "build>=1.0", - "pytest>=8.0", - "pytest-cov>=5.0", - "twine>=5.0", - "wheel>=0.41", - ], - "notebook": [ - "ipykernel>=6.0", - "ipywidgets>=8.0", - "jupyter>=1.0", - ], - }, - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - ], - keywords=[ - "data-leakage", - "machine-learning", - "permutation-test", - "scikit-learn", - ], -)