Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ pipeline so users can see the effect immediately.
pip install Leakly
```

For notebook environments that need the optional notebook dependencies:

```bash
pip install "Leakly[notebook]"
```

For the current GitHub checkout:

```bash
Expand Down Expand Up @@ -88,9 +94,7 @@ After label permutation, there should be no real biological, clinical, or statis

A leaky pipeline may still score well if information from the full dataset enters the analysis before the train/test split or outside the cross-validation loop. Common sources include feature selection, scaling, imputation, covariate adjustment, dimensionality reduction, or hyperparameter tuning performed on all samples.

This is especially problematic in high-dimensional data, such as neuroimaging, omics, or biomarker studies, where random label-specific patterns can appear meaningful by chance. If the test set influences preprocessing or feature selection, the model may preserve these random patterns and show inflated performance.

Leakly tests this directly: does the pipeline still perform above chance when labels are meaningless? If yes, it does not pinpoint the exact leakage source, but it strongly indicates that the pipeline needs inspection.
This is especially problematic in high-dimensional data, such as neuroimaging, omics, or biomarker studies, where random label-specific patterns can appear meaningful by chance. If the test set influences preprocessing or feature selection, the model may "remember" these random patterns and show inflated performance.

**How many permutations should I run?**

Expand Down
26 changes: 14 additions & 12 deletions example.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,22 @@
"metadata": {},
"outputs": [],
"source": [
"# Install Leakly locally, or from GitHub when running in Colab.\n",
"# Install Leakly from this checkout locally, or from PyPI elsewhere.\n",
"import importlib.metadata as metadata\n",
"from pathlib import Path\n",
"import subprocess\n",
"import sys\n",
"\n",
"install_args = [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\"]\n",
"try:\n",
" import google.colab # type: ignore # noqa: F401\n",
" install_target = \"git+https://github.com/DeMONLab-BioFINDER/Leakly.git@main\"\n",
" install_args.append(\"--force-reinstall\")\n",
"except ImportError:\n",
" install_target = \".[notebook]\"\n",
"repo_root = Path.cwd()\n",
"local_checkout = (repo_root / \"pyproject.toml\").exists() and (repo_root / \"leakly\").is_dir()\n",
"\n",
"subprocess.check_call([*install_args, install_target])\n",
"install_args = [sys.executable, \"-m\", \"pip\", \"install\", \"-q\"]\n",
"if local_checkout:\n",
" install_args.extend([\"-e\", \".[notebook]\"])\n",
"else:\n",
" install_args.extend([\"--upgrade\", \"Leakly\"])\n",
"\n",
"subprocess.check_call(install_args)\n",
"\n",
"for module_name in list(sys.modules):\n",
" if module_name == \"leakly\" or module_name.startswith(\"leakly.\"):\n",
Expand Down Expand Up @@ -169,10 +171,10 @@
"from leakly import permute_label\n",
"from tqdm.auto import tqdm\n",
"\n",
"# Use 2000 permutations for now to replicate `assets/AUC.png`.\n",
"# Use 100 permutations for a quick test run.\n",
"# Use 100 permutations for a quick example run.\n",
"# Increase to 2000 if you want to replicate `assets/AUC.png` more closely.\n",
"# Adjust based on your computational resources and needs.\n",
"N_PERMUTATIONS = 2000\n",
"N_PERMUTATIONS = 100\n",
"\n",
"leakage_permuted_scores = []\n",
"nonleakage_permuted_scores = []\n",
Expand Down
74 changes: 74 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
[build-system]
requires = ["setuptools>=77", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "Leakly"
dynamic = ["version"]
description = "Leakage checks for machine-learning pipelines using permutation tests."
readme = "README.md"
requires-python = ">=3.10"
license = "MIT"
license-files = ["LICENSE"]
authors = [
{ name = "DeMONLab-BioFINDER" },
]
keywords = [
"data-leakage",
"machine-learning",
"permutation-test",
"scikit-learn",
]
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Science/Research",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Topic :: Scientific/Engineering",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
]
dependencies = [
"joblib>=1.3,<2.0",
"matplotlib>=3.7,<4.0",
"numpy>=1.23,<2.4",
"pandas>=1.5,<3.0",
"PyYAML>=6.0,<7.0",
"scikit-learn>=1.6,<1.7",
"scipy>=1.9,<1.16",
"tqdm>=4.64,<5.0",
]

[project.optional-dependencies]
dev = [
"build>=1.0",
"pytest>=8.0",
"pytest-cov>=5.0",
"twine>=5.0",
"wheel>=0.41",
]
notebook = [
"ipykernel>=6.0",
"ipywidgets>=8.0",
"jupyter>=1.0",
]

[project.urls]
Homepage = "https://github.com/DeMONLab-BioFINDER/Leakly"
Source = "https://github.com/DeMONLab-BioFINDER/Leakly"
Issues = "https://github.com/DeMONLab-BioFINDER/Leakly/issues"

[tool.setuptools.dynamic]
version = { attr = "leakly.__version__" }

[tool.setuptools.packages.find]
include = ["leakly", "leakly.*"]

[tool.setuptools.package-data]
leakly = ["*.yaml"]

[tool.setuptools]
include-package-data = true
86 changes: 0 additions & 86 deletions setup.py

This file was deleted.

Loading