Upgrade to Ubuntu 24.04 LTS and Python 3.12 with modern uv dependency management

.github/workflows/ci.yml

@@ -11,11 +11,18 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          uv sync --frozen --extra dev
+          uv tool install tox --with tox-uv
 
       - name: Run CI
         run: |
-          pip install -r requirements/testing.txt
-          tox
+          uv run tox

.github/workflows/pip-tools.yaml

@@ -1,10 +1,8 @@
-# Action to run pip-compile weekly and create a Pull Request with the changes.
-# Although GitHub says that pip-compile is supported by dependabot, we couldn't make it work together.
-# That's why this action exists.
-# If we ever find the proper configuration for dependabot+pip-compile,
-# we can delete this action.
+# Action to run uv lock monthly and create a Pull Request with the changes.
+# This keeps dependencies up-to-date by updating uv.lock with the latest
+# compatible versions from pyproject.toml.
 
-name: Update dependencies with pip-tools
+name: Update dependencies with uv
 
 on:
   schedule:
@@ -24,30 +22,33 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
 
       - name: Install "pg_config" requirement
         run: sudo apt-get install libpq-dev
 
-      - name: Install piptools
-        run: python -m pip install -U pip-tools
-
-      - name: Update dependencies from requirements/*.txt
-        run: ./requirements/update.sh
+      - name: Update dependencies with uv lock
+        run: uv lock --upgrade
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
         with:
           add-paths: |
-            requirements/*.txt
+            uv.lock
           title: |
-            Dependencies: all packages updated via pip-tools
+            Dependencies: all packages updated via uv lock
           body: |
-            Dependencies: all packages updated via pip-tools
+            Dependencies: all packages updated via uv lock
+
+            This PR updates the `uv.lock` file with the latest compatible versions
+            of all dependencies specified in `pyproject.toml`.
           commit-message: |
-            Dependencies: all packages updated via pip-tools
+            Dependencies: all packages updated via uv lock
           delete-branch: true
-          branch: dependencies/pip-tools
+          branch: dependencies/uv-lock
           branch-suffix: short-commit-hash

.gitignore

@@ -95,3 +95,4 @@ geoip/*
 # ML models for topic classification
 trainingset-urls-cache.sqlite
 /machine_learning_experiments/assets/*.json
+.venv/

.pre-commit-config.yaml

@@ -7,10 +7,10 @@ repos:
   - id: trailing-whitespace
     exclude: '.*/dnt-policy.txt$'
 - repo: https://github.com/adamchainz/django-upgrade
-  rev: "1.14.1"
+  rev: "1.29.1"
   hooks:
   - id: django-upgrade
-    args: [--target-version, "4.2"]
+    args: [--target-version, "5.2"]
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
   rev: v0.5.2
@@ -20,3 +20,9 @@ repos:
       args: [ --fix ]
     # Run the formatter.
     - id: ruff-format
+- repo: https://github.com/astral-sh/uv-pre-commit
+  # uv version.
+  rev: 0.9.10
+  hooks:
+    # Compile requirements to ensure uv.lock is up-to-date
+    - id: uv-lock

Makefile

@@ -18,6 +18,8 @@ help:
 	@echo "  geoip          Download the GeoIP databases"
 	@echo "  ipproxy        Download proxy databases"
 
+# Run the full test suite
+# Run a specific test with tox -e py3 -- adserver/auth/tests.py
 test:
 	tox
 
@@ -46,11 +48,11 @@ dockerstop:
 # Use this command to inspect the container, run management commands,
 # or run anything else on the Django container
 dockershell:
-	docker compose -f $(DOCKER_CONFIG) run --rm django /bin/bash
+	docker compose -f $(DOCKER_CONFIG) run --rm django /shell
 
 # Get the GeoIP databases from DB-IP or Maxmind
 geoip:
-	python $(GEOIP_DOWNLOADER) --geoip-only --outdir=$(GEOIP_DIR)
+	uv run python $(GEOIP_DOWNLOADER) --geoip-only --outdir=$(GEOIP_DIR)
 
 ipproxy:
-	python $(GEOIP_DOWNLOADER) --ipproxy-only --outdir=$(GEOIP_DIR)
+	uv run python $(GEOIP_DOWNLOADER) --ipproxy-only --outdir=$(GEOIP_DIR)

docker-compose/django/Dockerfile

@@ -1,11 +1,19 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 ENV PYTHONUNBUFFERED 1
 
+# uv environment variables
+# Copy (don't hardlink) files into /.venv. Avoid issues with Docker's FS
+# https://docs.astral.sh/uv/reference/environment/
+ENV UV_LINK_MODE=copy
+ENV UV_PYTHON_DOWNLOADS=never
+ENV UV_PROJECT_ENVIRONMENT=/.venv
+
+
 RUN apt-get -y update
 RUN apt-get -y install \
         curl \
@@ -23,39 +31,48 @@ RUN apt-get -y install \
         libmysqlclient-dev \
         libfreetype6 \
         libjpeg-dev \
-        sqlite \
-        netcat \
+        sqlite3 \
+        netcat-openbsd \
         telnet \
         lsb-release
 
-# Requirements are installed here to ensure they will be cached.
-# https://docs.docker.com/build/cache/#use-the-dedicated-run-cache
-COPY ./requirements /requirements
-RUN pip install --upgrade pip
-RUN --mount=type=cache,target=/root/.cache/pip pip install -r /requirements/development.txt
-RUN --mount=type=cache,target=/root/.cache/pip pip install -r /requirements/production.txt
+# Install uv for fast package management
+# https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin/:$PATH"
+
+# Copy project files for dependency resolution
+# uv.lock ensures reproducible builds
+COPY pyproject.toml uv.lock ./
 
-# Comment this if you don't need the page/topic analyzer.
-# The analyzer is used to target ads better based on page content.
-# Its requirements are huge and include PyTorch and other ML tools.
-# If not needed, make sure to set `ADSERVER_ANALYZER_BACKEND=` (empty string)
-# in your environment file `./envs/local/django`.
-RUN --mount=type=cache,target=/root/.cache/pip pip install -r /requirements/analyzer.txt
+# Install dependencies using uv sync
+# This creates a virtual environment at /.venv (not in /app)
+# This installs all dependencies from uv.lock, ensuring consistency
+# --frozen: Don't update the lockfile
+# --no-install-project: Don't install the project itself
+# If you do not want the analyzer dependencies (which are LARGE), you can use:
+#  RUN --mount=type=cache,target=/root/.cache/uv \
+#      uv sync --frozen --no-install-project --extra dev --extra production
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-install-project --all-extras
 
 COPY ./docker-compose/django/start /start
 RUN chmod +x /start
 
+# Launch a shell within the container with this script
+COPY ./docker-compose/django/shell /shell
+RUN chmod +x /shell
+
 COPY ./docker-compose/django/celery/worker/start /start-celeryworker
 RUN chmod +x /start-celeryworker
 
 COPY ./docker-compose/django/celery/beat/start /start-celerybeat
 RUN chmod +x /start-celerybeat
 
-# Ensure that ``python`` is in the PATH so that ``./manage.py`` works
-RUN ln -s /usr/bin/python3 /usr/bin/python
-
 # Load model
-RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('multi-qa-MiniLM-L6-cos-v1', cache_folder='/tmp/sentence_transformers')"
+# Not needed if you don't use the analyzer
+RUN uv run python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('multi-qa-MiniLM-L6-cos-v1', cache_folder='/tmp/sentence_transformers')"
 
 # Setup the locale
 RUN locale-gen en_US.UTF-8

docker-compose/django/celery/beat/start

@@ -5,4 +5,4 @@ set -o nounset
 
 
 rm -f './celerybeat.pid'
-celery -A config.celery_app beat -l INFO
+uv run celery -A config.celery_app beat -l INFO

docker-compose/django/celery/worker/start

@@ -4,4 +4,4 @@ set -o errexit
 set -o nounset
 
 
-celery -A config.celery_app worker -l INFO -Q celery,analyzer,priority -c 1
+uv run celery -A config.celery_app worker -l INFO -Q celery,analyzer,priority -c 1

docker-compose/django/shell

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+# Use the venv created by `uv sync` in the Dockerfile
+source /.venv/bin/activate
+
+# Start shell with the venv activated
+/bin/bash

docker-compose/django/start

@@ -4,12 +4,13 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+
 # Reinstall dependencies without rebuilding docker image
 # pip install -r /app/requirements/development.txt
 
 # Don't auto-migrate locally because this can cause weird issues when testing migrations
 # python manage.py migrate
-python manage.py runserver 0.0.0.0:5000
+uv run ./manage.py runserver 0.0.0.0:5000
 
 # In production, we use gunicorn
 # This gets us close to that setting for development and testing

docs/conf.py

@@ -157,5 +157,5 @@
         "https://docs.djangoproject.com/en/dev/",
         "https://docs.djangoproject.com/en/dev/_objects/",
     ),
-    "python": ("https://docs.python.org/3.8/", None),
+    "python": ("https://docs.python.org/3.12/", None),
 }

docs/developer/quickstart.rst

@@ -43,8 +43,8 @@ This section is more to document steps than to encourage you to develop outside
 Requirements
 ~~~~~~~~~~~~
 
-- Python 3.10
-- Nodejs (tested with v14)
+- Python 3.12
+- Nodejs (tested with v20)
 
 Front-end assets
 ~~~~~~~~~~~~~~~~
@@ -57,10 +57,18 @@ To build the assets::
 Install Python dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+First, install uv if you haven't already:
+
+.. code-block:: bash
+
+   $ curl -LsSf https://astral.sh/uv/install.sh | sh
+
+Then install dependencies:
+
 .. code-block:: bash
 
-   $ pip install -r requirements/development.txt
-   $ pre-commit install            # Install a code style pre-commit hook
+   $ uv sync --all-extras           # Install all dependencies including dev tools
+   $ uv run pre-commit install      # Install a code style pre-commit hook
 
 Run the server
 ~~~~~~~~~~~~~~
@@ -69,19 +77,19 @@ Run migrations:
 
 .. code-block:: bash
 
-   $ python manage.py migrate
+   $ uv run ./manage.py migrate
 
 Create a superuser:
 
 .. code-block:: bash
 
-   $ python manage.py createsuperuser
+   $ uv run ./manage.py createsuperuser
 
 Run the server:
 
 .. code-block:: bash
 
-   $ python manage.py runserver
+   $ uv run ./manage.py runserver
 
 Running the tests
 -----------------
@@ -90,8 +98,8 @@ To run the unit tests:
 
 .. code-block:: bash
 
-    $ pip install -r requirements/testing.txt
-    $ make test
+    $ uv tool install tox --with tox-uv
+    $ tox
 
 Run a specific test: