From d0a8fc5ff040e7b2db249382f4dad4394f20e74f Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sun, 22 Feb 2026 13:50:08 -0800 Subject: [PATCH 01/32] update server version to 2.4.4 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 061b099..403ed31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ license = { file = "LICENSE" } authors = [ { name = "psilabs-dev" } ] -version = "2.4.3" +version = "2.4.4" dependencies = [ "aiofiles>=25.1.0", "celery>=5.5.3", diff --git a/uv.lock b/uv.lock index 410baaf..9cc0790 100644 --- a/uv.lock +++ b/uv.lock @@ -896,7 +896,7 @@ wheels = [ [[package]] name = "pixivutil-server" -version = "2.4.3" +version = "2.4.4" source = { editable = "." } dependencies = [ { name = "aiofiles" }, From 25b51d777279ff052c50ceddf3a5d84bc611d126 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sun, 22 Feb 2026 13:51:04 -0800 Subject: [PATCH 02/32] update PixivUtil2 to address https://github.com/psilabs-dev/PixivUtil2/issues/37 --- PixivUtil2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PixivUtil2 b/PixivUtil2 index 2943173..007e657 160000 --- a/PixivUtil2 +++ b/PixivUtil2 @@ -1 +1 @@ -Subproject commit 2943173256420eb191bf7be013fb674d52cca89d +Subproject commit 007e65778b08d7fdc1eae5d54bea61d923ac739e From 7daacca27c47b4f005eb662a2f75daf54cafc3c8 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Mon, 23 Feb 2026 18:01:01 -0800 Subject: [PATCH 03/32] stop pyright from complaining --- pyproject.toml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 403ed31..e7eeb89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,14 @@ pixivutil2 = [ [tool.pyright] pythonVersion = "3.12" include = ["PixivServer", "PixivServerCommon", "PixivUtilClient", "tests"] -exclude = ["PixivUtil2"] +exclude = [ + "PixivUtil2", + "**/node_modules", + "**/__pycache__", + "**/.*", ".venv", +] reportUnusedCoroutine = "error" +extraPaths = ["PixivUtil2"] [tool.ruff] exclude = [ From 6c29501b56726357571c6606706b6220e5589c81 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Mon, 23 Feb 2026 20:58:25 -0800 Subject: [PATCH 04/32] refactor out metrics and add environment configurations --- PixivServer/app.py | 127 ++----------------------------- PixivServer/config/server.py | 6 ++ PixivServer/service/metrics.py | 134 +++++++++++++++++++++++++++++++++ README.md | 2 + docker-compose.yml | 6 +- 5 files changed, 151 insertions(+), 124 deletions(-) create mode 100644 PixivServer/service/metrics.py diff --git a/PixivServer/app.py b/PixivServer/app.py index 3b18382..300e644 100644 --- a/PixivServer/app.py +++ b/PixivServer/app.py @@ -1,18 +1,9 @@ import asyncio -import base64 -import contextlib -import json import logging -import os import time import traceback -import urllib.error -import urllib.request from contextlib import asynccontextmanager -from pathlib import Path -from urllib.parse import quote, urlparse -import psutil from fastapi import Depends, FastAPI, Request, Response import PixivServer @@ -28,132 +19,24 @@ # import PixivServer.routers.subscription import PixivServer.service import PixivServer.service.pixiv -from PixivServer.config.pixivutil import config as pixivutil_config -from PixivServer.config.rabbitmq import config as rabbitmq_config +from PixivServer.config.server import config as server_config from PixivServer.metrics import ( - DB_ARTWORKS, - DB_MEMBERS, - DB_PAGES, - DB_SERIES, - DB_TAGS, - DISK_DATABASE_BYTES, - DISK_DOWNLOADS_BYTES, HTTP_REQUEST_DURATION, HTTP_REQUEST_SIZE, HTTP_REQUESTS_TOTAL, HTTP_RESPONSE_SIZE, - QUEUE_DEPTH, SERVER_INFO, - SYS_CPU_PERCENT, - SYS_DISK_TOTAL_BYTES, - SYS_DISK_USED_BYTES, - SYS_MEM_TOTAL_BYTES, - SYS_MEM_USED_BYTES, ) -from PixivServer.repository.pixivutil import PixivUtilRepository +from PixivServer.service.metrics import periodic_metrics_collector from PixivServer.utils import get_version logger = logging.getLogger('uvicorn.pixivutil') -_SYSTEM_COLLECT_INTERVAL = 15 # seconds -_DB_STAT_COLLECT_INTERVAL = 60 # seconds -_DISK_COLLECT_INTERVAL = 300 # seconds (directory walk may be slow on large collections) -_QUEUE_COLLECT_INTERVAL = 15 # seconds - - -def _collect_system_metrics() -> None: - cpu = psutil.cpu_percent(interval=1) - vm = psutil.virtual_memory() - disk = psutil.disk_usage("/") - SYS_CPU_PERCENT.set(cpu) - SYS_MEM_USED_BYTES.set(vm.used) - SYS_MEM_TOTAL_BYTES.set(vm.total) - SYS_DISK_USED_BYTES.set(disk.used) - SYS_DISK_TOTAL_BYTES.set(disk.total) - - -def _collect_db_stats() -> None: - repo = PixivUtilRepository() - repo.open() - try: - DB_MEMBERS.set(repo.count_members()) - DB_ARTWORKS.set(repo.count_artworks()) - DB_PAGES.set(repo.count_pages()) - DB_TAGS.set(repo.count_tags()) - DB_SERIES.set(repo.count_series()) - finally: - repo.close() - - -def _collect_disk_metrics() -> None: - # Database file + WAL/SHM sidecars - db_path = pixivutil_config.db_path - db_bytes = 0 - for suffix in ("", "-wal", "-shm"): - p = db_path + suffix - if os.path.isfile(p): - db_bytes += os.path.getsize(p) - DISK_DATABASE_BYTES.set(db_bytes) - - # Downloads directory — recursive file size sum - downloads = Path(PixivServer.service.pixiv.service.downloads_folder) - total = 0 - if downloads.is_dir(): - for f in downloads.rglob("*"): - if f.is_file(): - with contextlib.suppress(OSError): - total += f.stat().st_size - DISK_DOWNLOADS_BYTES.set(total) - - -def _collect_queue_depth() -> None: - parsed = urlparse(rabbitmq_config.broker_url) - user = parsed.username or "guest" - password = parsed.password or "guest" - host = parsed.hostname or "rabbitmq" - raw_vhost = parsed.path.lstrip("/") - vhost = raw_vhost if raw_vhost else "/" - encoded_vhost = quote(vhost, safe="") - url = f"http://{host}:15672/api/queues/{encoded_vhost}/pixivutil-queue" - credentials = base64.b64encode(f"{user}:{password}".encode()).decode() - req = urllib.request.Request(url, headers={"Authorization": f"Basic {credentials}"}) - try: - with urllib.request.urlopen(req, timeout=5) as resp: - data = json.loads(resp.read()) - QUEUE_DEPTH.set(data.get("messages", 0)) - except (urllib.error.URLError, OSError, ValueError): - pass # Management API unavailable; leave metric stale - - -async def _periodic_metrics_collector() -> None: - last_system = 0.0 - last_db = 0.0 - last_disk = 0.0 - last_queue = 0.0 - while True: - now = time.monotonic() - try: - if now - last_system >= _SYSTEM_COLLECT_INTERVAL: - await asyncio.to_thread(_collect_system_metrics) - last_system = time.monotonic() - if now - last_db >= _DB_STAT_COLLECT_INTERVAL: - await asyncio.to_thread(_collect_db_stats) - last_db = time.monotonic() - if now - last_disk >= _DISK_COLLECT_INTERVAL: - await asyncio.to_thread(_collect_disk_metrics) - last_disk = time.monotonic() - if now - last_queue >= _QUEUE_COLLECT_INTERVAL: - await asyncio.to_thread(_collect_queue_depth) - last_queue = time.monotonic() - except Exception: # noqa: BLE001 - logger.warning(f"Metrics collector error: {traceback.format_exc()}") - await asyncio.sleep(1) - - @asynccontextmanager async def lifespan(_: FastAPI): try: - logger.info("Setting up server.") + logger.info(f"Setting up server in {server_config.server_env} environment.") + # startup actions await asyncio.sleep(5) PixivServer.service.pixiv.service.open(validate_pixiv_login=False) @@ -162,7 +45,7 @@ async def lifespan(_: FastAPI): except Exception as e: print(f"Encountered exception during application setup: {traceback.format_exc()}") raise e - collector_task = asyncio.create_task(_periodic_metrics_collector()) + collector_task = asyncio.create_task(periodic_metrics_collector()) yield # shutdown actions collector_task.cancel() diff --git a/PixivServer/config/server.py b/PixivServer/config/server.py index cbbf641..8ed690c 100644 --- a/PixivServer/config/server.py +++ b/PixivServer/config/server.py @@ -1,4 +1,5 @@ import os +from typing import Literal class ServerConfig: @@ -8,4 +9,9 @@ def __init__(self): api_key = os.getenv("PIXIVUTIL_SERVER_API_KEY") self.api_key = api_key if api_key else None + server_env = os.getenv("PIXIVUTIL_SERVER_ENV", "production") + if server_env != "production" and server_env != "development": + raise ValueError(f"Unrecognized environment: {server_env}") + self.server_env: Literal["production", "development"] = server_env + config = ServerConfig() diff --git a/PixivServer/service/metrics.py b/PixivServer/service/metrics.py new file mode 100644 index 0000000..93883f8 --- /dev/null +++ b/PixivServer/service/metrics.py @@ -0,0 +1,134 @@ +import asyncio +import base64 +import contextlib +import json +import logging +import os +import time +import traceback +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import quote, urlparse + +import psutil + +import PixivServer + +# import PixivServer.routers.subscription +import PixivServer.service +import PixivServer.service.pixiv +from PixivServer.config.pixivutil import config as pixivutil_config +from PixivServer.config.rabbitmq import config as rabbitmq_config +from PixivServer.metrics import ( + DB_ARTWORKS, + DB_MEMBERS, + DB_PAGES, + DB_SERIES, + DB_TAGS, + DISK_DATABASE_BYTES, + DISK_DOWNLOADS_BYTES, + QUEUE_DEPTH, + SYS_CPU_PERCENT, + SYS_DISK_TOTAL_BYTES, + SYS_DISK_USED_BYTES, + SYS_MEM_TOTAL_BYTES, + SYS_MEM_USED_BYTES, +) +from PixivServer.repository.pixivutil import PixivUtilRepository + +logger = logging.getLogger('uvicorn.pixivutil') + +_SYSTEM_COLLECT_INTERVAL = 15 # seconds +_DB_STAT_COLLECT_INTERVAL = 60 # seconds +_DISK_COLLECT_INTERVAL = 300 # seconds (directory walk may be slow on large collections) +_QUEUE_COLLECT_INTERVAL = 15 # seconds + + +def _collect_system_metrics() -> None: + cpu = psutil.cpu_percent(interval=1) + vm = psutil.virtual_memory() + disk = psutil.disk_usage("/") + SYS_CPU_PERCENT.set(cpu) + SYS_MEM_USED_BYTES.set(vm.used) + SYS_MEM_TOTAL_BYTES.set(vm.total) + SYS_DISK_USED_BYTES.set(disk.used) + SYS_DISK_TOTAL_BYTES.set(disk.total) + + +def _collect_db_stats() -> None: + repo = PixivUtilRepository() + repo.open() + try: + DB_MEMBERS.set(repo.count_members()) + DB_ARTWORKS.set(repo.count_artworks()) + DB_PAGES.set(repo.count_pages()) + DB_TAGS.set(repo.count_tags()) + DB_SERIES.set(repo.count_series()) + finally: + repo.close() + + +def _collect_disk_metrics() -> None: + # Database file + WAL/SHM sidecars + db_path = pixivutil_config.db_path + db_bytes = 0 + for suffix in ("", "-wal", "-shm"): + p = db_path + suffix + if os.path.isfile(p): + db_bytes += os.path.getsize(p) + DISK_DATABASE_BYTES.set(db_bytes) + + # Downloads directory — recursive file size sum + downloads = Path(PixivServer.service.pixiv.service.downloads_folder) + total = 0 + if downloads.is_dir(): + for f in downloads.rglob("*"): + if f.is_file(): + with contextlib.suppress(OSError): + total += f.stat().st_size + DISK_DOWNLOADS_BYTES.set(total) + + +def _collect_queue_depth() -> None: + parsed = urlparse(rabbitmq_config.broker_url) + user = parsed.username or "guest" + password = parsed.password or "guest" + host = parsed.hostname or "rabbitmq" + raw_vhost = parsed.path.lstrip("/") + vhost = raw_vhost if raw_vhost else "/" + encoded_vhost = quote(vhost, safe="") + url = f"http://{host}:15672/api/queues/{encoded_vhost}/pixivutil-queue" # TODO: if rabbitmq management layer isn't set up, then we have a problem. + credentials = base64.b64encode(f"{user}:{password}".encode()).decode() + req = urllib.request.Request(url, headers={"Authorization": f"Basic {credentials}"}) + try: + with urllib.request.urlopen(req, timeout=5) as resp: + data = json.loads(resp.read()) + QUEUE_DEPTH.set(data.get("messages", 0)) + except (urllib.error.URLError, OSError, ValueError): + pass # Management API unavailable; leave metric stale + + +async def periodic_metrics_collector() -> None: + last_system = 0.0 + last_db = 0.0 + last_disk = 0.0 + last_queue = 0.0 + while True: + now = time.monotonic() + try: + if now - last_system >= _SYSTEM_COLLECT_INTERVAL: + await asyncio.to_thread(_collect_system_metrics) + last_system = time.monotonic() + if now - last_db >= _DB_STAT_COLLECT_INTERVAL: + await asyncio.to_thread(_collect_db_stats) + last_db = time.monotonic() + if now - last_disk >= _DISK_COLLECT_INTERVAL: + await asyncio.to_thread(_collect_disk_metrics) + last_disk = time.monotonic() + if now - last_queue >= _QUEUE_COLLECT_INTERVAL: + await asyncio.to_thread(_collect_queue_depth) + last_queue = time.monotonic() + except Exception: # noqa: BLE001 + logger.warning(f"Metrics collector error: {traceback.format_exc()}") + await asyncio.sleep(1) diff --git a/README.md b/README.md index 953dd54..8b69ae3 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ Authorization: Bearer ## Architecture and Development +When running PixivUtil server in development, set `PIXIVUTIL_SERVER_ENV=development`. This will enable debug logging level. + PixivUtil server is a Python project based on PixivUtil2 as its API client engine. PixivUtil2 is a separate git repository added to this as a submodule. PixivUtil server as a service consists of 3 microservices: the PixivUtil API server, PixivUtil worker, and RabbitMQ queue. The server receives API requests from the user/client, and passes them as long-running jobs to a single-process worker which handles them one at a time via the queue, controlling API volumes and avoiding rate limit violations. diff --git a/docker-compose.yml b/docker-compose.yml index 0be6051..df14b52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,12 +27,13 @@ services: build: . container_name: pixivutil-worker # entrypoint: ["uv", "run"] # use this if you want to keep a non-root user. - command: ["celery", "-A", "PixivServer.worker.pixiv_worker", "worker", "--concurrency=1", "--loglevel=info", "-B"] + command: ["celery", "-A", "PixivServer.worker.pixiv_worker", "worker", "--concurrency=1", "--loglevel=debug", "-B"] environment: - PUID=${PUID:-1000} - PGID=${PGID:-1000} - CELERYBEAT_SCHEDULE=/workdir/.pixivUtil2/celerybeat-schedule - PIXIVUTIL_COOKIE=$PIXIVUTIL_COOKIE + - PIXIVUTIL_SERVER_ENV=development - RABBITMQ_BROKER_URL=amqp://guest:guest@rabbitmq:5672 volumes: - pixivutil-data:/workdir/.pixivUtil2 @@ -45,12 +46,13 @@ services: build: . container_name: pixivutil-server # entrypoint: ["uv", "run"] # use this if you want to keep a non-root user. - command: ["uvicorn", "PixivServer.app:app", "--host", "0.0.0.0", "--port", "8000", "--log-level", "info"] + command: ["uvicorn", "PixivServer.app:app", "--host", "0.0.0.0", "--port", "8000", "--log-level", "debug"] environment: - PUID=${PUID:-1000} - PGID=${PGID:-1000} - PIXIVUTIL_COOKIE=$PIXIVUTIL_COOKIE - PIXIVUTIL_SERVER_API_KEY=$PIXIVUTIL_SERVER_API_KEY + - PIXIVUTIL_SERVER_ENV=development - RABBITMQ_BROKER_URL=amqp://guest:guest@rabbitmq:5672 volumes: - pixivutil-data:/workdir/.pixivUtil2 From 3e764dc1c4d65cb558348e6e02de8cfbaa9773b1 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Mon, 23 Feb 2026 21:01:34 -0800 Subject: [PATCH 05/32] update dockerignore to exclude new gui files --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index d70a602..795a1d1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -30,6 +30,7 @@ PixivUtil2/.pylintrc PixivUtil2/.python-version PixivUtil2/.travis.yml PixivUtil2/Dockerfile +PixivUtil2/PixivUtilGUI.py PixivUtil2/ISSUE_TEMPLATE.md PixivUtil2/MANIFEST.in PixivUtil2/changelog.txt @@ -37,6 +38,7 @@ PixivUtil2/freeimage-3.15.4-win32.dll PixivUtil2/icon2.ico PixivUtil2/pyproject.toml PixivUtil2/readme.md +PixivUtil2/README.md PixivUtil2/requirements.txt PixivUtil2/setup.py PixivUtil2/uv.lock From 2eb03e11fe9b066db5366bbe258c9426983eca0f Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Mon, 23 Feb 2026 21:34:43 -0800 Subject: [PATCH 06/32] update PixivUtil2 to server mode version 0.2.0 --- PixivUtil2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PixivUtil2 b/PixivUtil2 index 007e657..68b6d7e 160000 --- a/PixivUtil2 +++ b/PixivUtil2 @@ -1 +1 @@ -Subproject commit 007e65778b08d7fdc1eae5d54bea61d923ac739e +Subproject commit 68b6d7e37e9448f746de49edce89d6c1865ff472 From ef3aa959cd52eed65c5fde4c60a2d74ad50af032 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Mon, 23 Feb 2026 22:26:34 -0800 Subject: [PATCH 07/32] refactor server and worker and add dev mode --- PixivServer/app.py | 8 + PixivServer/routers/dev.py | 28 ++++ PixivServer/routers/download_queue.py | 2 +- PixivServer/routers/metadata_queue.py | 2 +- PixivServer/worker.py | 201 -------------------------- PixivServer/worker/__init__.py | 61 ++++++++ PixivServer/worker/dev.py | 32 ++++ PixivServer/worker/download.py | 81 +++++++++++ PixivServer/worker/metadata.py | 93 ++++++++++++ 9 files changed, 305 insertions(+), 203 deletions(-) create mode 100644 PixivServer/routers/dev.py delete mode 100644 PixivServer/worker.py create mode 100644 PixivServer/worker/__init__.py create mode 100644 PixivServer/worker/dev.py create mode 100644 PixivServer/worker/download.py create mode 100644 PixivServer/worker/metadata.py diff --git a/PixivServer/app.py b/PixivServer/app.py index 300e644..a49ca79 100644 --- a/PixivServer/app.py +++ b/PixivServer/app.py @@ -122,6 +122,14 @@ async def request_metrics_middleware(request: Request, call_next): # prefix="/api/subscription" # ) +if server_config.server_env == 'development': + import PixivServer.routers.dev + app.include_router( + PixivServer.routers.dev.router, + prefix="/api/dev", + dependencies=auth_dependency, + ) + @app.get("/") async def info(): diff --git a/PixivServer/routers/dev.py b/PixivServer/routers/dev.py new file mode 100644 index 0000000..dfc081f --- /dev/null +++ b/PixivServer/routers/dev.py @@ -0,0 +1,28 @@ +import logging + +from celery.result import AsyncResult +from fastapi import APIRouter, Response +from fastapi.responses import JSONResponse + +from PixivServer.models.pixiv_worker import DownloadArtworkByIdRequest +from PixivServer.worker.dev import dev_download_artworks_by_id + +logger = logging.getLogger('uvicorn.pixivutil') +router = APIRouter() + + +@router.post("/artwork/{artwork_id}") +async def dev_queue_download_artwork_by_id(artwork_id: str) -> Response: + """ + (test) Download Pixiv image by ID. + """ + logger.info(f"Downloading Pixiv artwork by image ID: {artwork_id}.") + request = DownloadArtworkByIdRequest(artwork_id=int(artwork_id)) + artwork_title, member_name = "artwork title", "member title" + task: AsyncResult = dev_download_artworks_by_id.delay(request.model_dump()) + return JSONResponse({ + "task_id": task.id, + 'artwork_id': artwork_id, + "artwork_title": artwork_title, + "member_name": member_name, + }) diff --git a/PixivServer/routers/download_queue.py b/PixivServer/routers/download_queue.py index e5fa051..870b574 100644 --- a/PixivServer/routers/download_queue.py +++ b/PixivServer/routers/download_queue.py @@ -17,7 +17,7 @@ ) from PixivServer.repository.pixivutil import PixivUtilRepository from PixivServer.utils import is_valid_date -from PixivServer.worker import ( +from PixivServer.worker.download import ( delete_artwork_by_id, download_artworks_by_id, download_artworks_by_member_id, diff --git a/PixivServer/routers/metadata_queue.py b/PixivServer/routers/metadata_queue.py index 017b267..1ac1fe7 100644 --- a/PixivServer/routers/metadata_queue.py +++ b/PixivServer/routers/metadata_queue.py @@ -12,7 +12,7 @@ DownloadSeriesMetadataByIdRequest, DownloadTagMetadataByIdRequest, ) -from PixivServer.worker import ( +from PixivServer.worker.metadata import ( download_artwork_metadata_by_id, download_member_metadata_by_id, download_series_metadata_by_id, diff --git a/PixivServer/worker.py b/PixivServer/worker.py deleted file mode 100644 index 432bcec..0000000 --- a/PixivServer/worker.py +++ /dev/null @@ -1,201 +0,0 @@ -import logging -import random -import time -import traceback - -from celery import Celery -from celery.signals import setup_logging, worker_init, worker_shutdown - -import PixivServer - -# from PixivServer.config.worker import config as worker_config -import PixivServer.service -import PixivServer.service.pixiv -from PixivServer.models.pixiv_worker import ( - DeleteArtworkByIdRequest, - DownloadArtworkByIdRequest, - DownloadArtworkMetadataByIdRequest, - DownloadArtworksByMemberIdRequest, - DownloadArtworksByTagsRequest, - DownloadMemberMetadataByIdRequest, - DownloadSeriesMetadataByIdRequest, - DownloadTagMetadataByIdRequest, -) - -logger = logging.getLogger(__name__) - -pixiv_worker = Celery(__name__) -pixiv_worker.config_from_object('PixivServer.config.celery') - -def __job_sleep(): - """ - Sleep a random interval between 1-5s for all jobs. - Synchronous/blocking sleep. - """ - time_to_sleep = random.uniform(1, 5) - time.sleep(time_to_sleep) - return 0 - -@worker_init.connect -def on_worker_init(*args, **kwargs): - PixivServer.service.pixiv.service.open() - return - -@worker_shutdown.connect -def on_worker_shutdown(*args, **kwargs): - PixivServer.service.pixiv.service.close() - return - -@setup_logging.connect -def config_loggers(*args, **kwargs): - return - -# @celery.on_after_configure.connect -# def setup_periodic_tasks(sender, **kwargs): -# sender.add_periodic_task(worker_config.subscription_time_seconds, run_artist_subscription_job.s(), name='Artist subscription job') -# sender.add_periodic_task(worker_config.subscription_time_seconds, run_tag_subscription_job.s(), name='Tag subscription job') - -# @celery.task(name='run_artist_subscription_job') -# def run_artist_subscription_job(): -# logger.info('Running scheduled member subscription job...') -# new_artworks_by_member_names = subscription_service.run_member_subscription_job() -# member_names = list(new_artworks_by_member_names.keys()) -# if member_names: -# message = '[Scheduled job]: Downloaded new artworks from: ' + ', '.join(member_names) -# return True - -# @celery.task(name='run_tag_subscription_job') -# def run_tag_subscription_job(): -# ''' -# Since this is calling process_tags directly cannot extract logs. -# ''' -# logger.info('Running scheduled tag subscription job...') -# subscription_service.run_tag_subscription_job() - -@pixiv_worker.task(name="download_artworks_by_id", queue='pixivutil-queue') -def download_artworks_by_id(request_dict: dict): - try: - request = DownloadArtworkByIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by ID: {request.artwork_id}.") - PixivServer.service.pixiv.service.download_artwork_by_id(request) - return True - except Exception as e: - logger.error(f"Error in download_artworks_by_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() - -@pixiv_worker.task(name="download_artworks_by_member_id", queue='pixivutil-queue') -def download_artworks_by_member_id(request_dict: dict): - try: - request = DownloadArtworksByMemberIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artworks by member ID: {request.member_id}.") - PixivServer.service.pixiv.service.download_artworks_by_member_id(request) - return True - except Exception as e: - logger.error(f"Error in download_artworks_by_member_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() - -@pixiv_worker.task(name="download_artworks_by_tag", queue='pixivutil-queue') -def download_artworks_by_tag(request_dict: dict): - try: - request = DownloadArtworksByTagsRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by tag: {request.tags}. Bookmark minimum: {request.bookmark_count}") - PixivServer.service.pixiv.service.download_artworks_by_tag(request) - return True - except Exception as e: - logger.error(f"Error in download_artworks_by_tag worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() - -@pixiv_worker.task(name="delete_artwork_by_id", queue='pixivutil-queue') -def delete_artwork_by_id(request_dict: dict): - try: - request = DeleteArtworkByIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Deleting artwork by ID: {request.artwork_id}.") - PixivServer.service.pixiv.service.delete_artwork_by_id(request) - return True - except Exception as e: - logger.error(f"Error in delete_artwork_by_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() - - -@pixiv_worker.task(name="download_member_metadata_by_id", queue='pixivutil-queue') -def download_member_metadata_by_id(request_dict: dict): - try: - request = DownloadMemberMetadataByIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log( - "info", - f"Downloading member metadata by ID: {request.member_id}.", - ) - PixivServer.service.pixiv.service.download_member_metadata_by_id(request) - return True - except Exception as e: - logger.error(f"Error in download_member_metadata_by_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() - - -@pixiv_worker.task(name="download_artwork_metadata_by_id", queue='pixivutil-queue') -def download_artwork_metadata_by_id(request_dict: dict): - try: - request = DownloadArtworkMetadataByIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log( - "info", - f"Downloading artwork metadata by ID: {request.artwork_id}.", - ) - PixivServer.service.pixiv.service.download_artwork_metadata_by_id(request) - return True - except Exception as e: - logger.error(f"Error in download_artwork_metadata_by_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() - - -@pixiv_worker.task(name="download_series_metadata_by_id", queue='pixivutil-queue') -def download_series_metadata_by_id(request_dict: dict): - try: - request = DownloadSeriesMetadataByIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log( - "info", - f"Downloading series metadata by ID: {request.series_id}.", - ) - PixivServer.service.pixiv.service.download_series_metadata_by_id(request) - return True - except Exception as e: - logger.error(f"Error in download_series_metadata_by_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() - - -@pixiv_worker.task(name="download_tag_metadata_by_id", queue='pixivutil-queue') -def download_tag_metadata_by_id(request_dict: dict): - try: - request = DownloadTagMetadataByIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log( - "info", - f"Downloading tag metadata: {request.tag} (filter_mode={request.filter_mode}).", - ) - PixivServer.service.pixiv.service.download_tag_metadata_by_id(request) - return True - except Exception as e: - logger.error(f"Error in download_tag_metadata_by_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() diff --git a/PixivServer/worker/__init__.py b/PixivServer/worker/__init__.py new file mode 100644 index 0000000..98a4c33 --- /dev/null +++ b/PixivServer/worker/__init__.py @@ -0,0 +1,61 @@ +import logging + +from celery import Celery +from celery.signals import setup_logging, worker_init, worker_shutdown + +import PixivServer +import PixivServer.service +import PixivServer.service.pixiv +from PixivServer.config.server import config as server_config + +logger = logging.getLogger(__name__) + +pixiv_worker = Celery(__name__) +pixiv_worker.config_from_object('PixivServer.config.celery') + + +@worker_init.connect +def on_worker_init(*args, **kwargs): + PixivServer.service.pixiv.service.open() + return + + +@worker_shutdown.connect +def on_worker_shutdown(*args, **kwargs): + PixivServer.service.pixiv.service.close() + return + + +@setup_logging.connect +def config_loggers(*args, **kwargs): + return + + +# @celery.on_after_configure.connect +# def setup_periodic_tasks(sender, **kwargs): +# sender.add_periodic_task(worker_config.subscription_time_seconds, run_artist_subscription_job.s(), name='Artist subscription job') +# sender.add_periodic_task(worker_config.subscription_time_seconds, run_tag_subscription_job.s(), name='Tag subscription job') + +# @celery.task(name='run_artist_subscription_job') +# def run_artist_subscription_job(): +# logger.info('Running scheduled member subscription job...') +# new_artworks_by_member_names = subscription_service.run_member_subscription_job() +# member_names = list(new_artworks_by_member_names.keys()) +# if member_names: +# message = '[Scheduled job]: Downloaded new artworks from: ' + ', '.join(member_names) +# return True + +# @celery.task(name='run_tag_subscription_job') +# def run_tag_subscription_job(): +# ''' +# Since this is calling process_tags directly cannot extract logs. +# ''' +# logger.info('Running scheduled tag subscription job...') +# subscription_service.run_tag_subscription_job() + +# Register task modules +import PixivServer.worker.download # noqa: E402, F401 +import PixivServer.worker.metadata # noqa: E402, F401 + +if server_config.server_env == 'development': + import PixivServer.worker.dev # noqa: F401 diff --git a/PixivServer/worker/dev.py b/PixivServer/worker/dev.py new file mode 100644 index 0000000..a9e482f --- /dev/null +++ b/PixivServer/worker/dev.py @@ -0,0 +1,32 @@ +import logging +import random +import time +import traceback + +from celery import shared_task + +import PixivServer.service.pixiv +from PixivServer.models.pixiv_worker import DownloadArtworkByIdRequest + +logger = logging.getLogger(__name__) + + +def __job_sleep(): + time.sleep(random.uniform(1, 5)) + return 0 + + +@shared_task(name="dev_download_artworks_by_id", queue='pixivutil-queue') +def dev_download_artworks_by_id(request_dict: dict): + try: + request = DownloadArtworkByIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"(test) Downloading artwork by ID: {request.artwork_id}.") + time.sleep(3) + PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"(test) Completed download artwork by ID: {request.artwork_id}.") + return True + except Exception as e: + logger.error(f"Error in download_artworks_by_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py new file mode 100644 index 0000000..f4449d1 --- /dev/null +++ b/PixivServer/worker/download.py @@ -0,0 +1,81 @@ +import logging +import random +import time +import traceback + +from celery import shared_task + +import PixivServer.service.pixiv +from PixivServer.models.pixiv_worker import ( + DeleteArtworkByIdRequest, + DownloadArtworkByIdRequest, + DownloadArtworksByMemberIdRequest, + DownloadArtworksByTagsRequest, +) + +logger = logging.getLogger(__name__) + + +def __job_sleep(): + time.sleep(random.uniform(1, 5)) + return 0 + + +@shared_task(name="download_artworks_by_id", queue='pixivutil-queue') +def download_artworks_by_id(request_dict: dict): + try: + request = DownloadArtworkByIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by ID: {request.artwork_id}.") + PixivServer.service.pixiv.service.download_artwork_by_id(request) + return True + except Exception as e: + logger.error(f"Error in download_artworks_by_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() + + +@shared_task(name="download_artworks_by_member_id", queue='pixivutil-queue') +def download_artworks_by_member_id(request_dict: dict): + try: + request = DownloadArtworksByMemberIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artworks by member ID: {request.member_id}.") + PixivServer.service.pixiv.service.download_artworks_by_member_id(request) + return True + except Exception as e: + logger.error(f"Error in download_artworks_by_member_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() + + +@shared_task(name="download_artworks_by_tag", queue='pixivutil-queue') +def download_artworks_by_tag(request_dict: dict): + try: + request = DownloadArtworksByTagsRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by tag: {request.tags}. Bookmark minimum: {request.bookmark_count}") + PixivServer.service.pixiv.service.download_artworks_by_tag(request) + return True + except Exception as e: + logger.error(f"Error in download_artworks_by_tag worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() + + +@shared_task(name="delete_artwork_by_id", queue='pixivutil-queue') +def delete_artwork_by_id(request_dict: dict): + try: + request = DeleteArtworkByIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Deleting artwork by ID: {request.artwork_id}.") + PixivServer.service.pixiv.service.delete_artwork_by_id(request) + return True + except Exception as e: + logger.error(f"Error in delete_artwork_by_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py new file mode 100644 index 0000000..bf23298 --- /dev/null +++ b/PixivServer/worker/metadata.py @@ -0,0 +1,93 @@ +import logging +import random +import time +import traceback + +from celery import shared_task + +import PixivServer.service.pixiv +from PixivServer.models.pixiv_worker import ( + DownloadArtworkMetadataByIdRequest, + DownloadMemberMetadataByIdRequest, + DownloadSeriesMetadataByIdRequest, + DownloadTagMetadataByIdRequest, +) + +logger = logging.getLogger(__name__) + + +def __job_sleep(): + time.sleep(random.uniform(1, 5)) + return 0 + + +@shared_task(name="download_member_metadata_by_id", queue='pixivutil-queue') +def download_member_metadata_by_id(request_dict: dict): + try: + request = DownloadMemberMetadataByIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log( + "info", + f"Downloading member metadata by ID: {request.member_id}.", + ) + PixivServer.service.pixiv.service.download_member_metadata_by_id(request) + return True + except Exception as e: + logger.error(f"Error in download_member_metadata_by_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() + + +@shared_task(name="download_artwork_metadata_by_id", queue='pixivutil-queue') +def download_artwork_metadata_by_id(request_dict: dict): + try: + request = DownloadArtworkMetadataByIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log( + "info", + f"Downloading artwork metadata by ID: {request.artwork_id}.", + ) + PixivServer.service.pixiv.service.download_artwork_metadata_by_id(request) + return True + except Exception as e: + logger.error(f"Error in download_artwork_metadata_by_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() + + +@shared_task(name="download_series_metadata_by_id", queue='pixivutil-queue') +def download_series_metadata_by_id(request_dict: dict): + try: + request = DownloadSeriesMetadataByIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log( + "info", + f"Downloading series metadata by ID: {request.series_id}.", + ) + PixivServer.service.pixiv.service.download_series_metadata_by_id(request) + return True + except Exception as e: + logger.error(f"Error in download_series_metadata_by_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() + + +@shared_task(name="download_tag_metadata_by_id", queue='pixivutil-queue') +def download_tag_metadata_by_id(request_dict: dict): + try: + request = DownloadTagMetadataByIdRequest(**request_dict) + PixivServer.service.pixiv.PixivHelper.print_and_log( + "info", + f"Downloading tag metadata: {request.tag} (filter_mode={request.filter_mode}).", + ) + PixivServer.service.pixiv.service.download_tag_metadata_by_id(request) + return True + except Exception as e: + logger.error(f"Error in download_tag_metadata_by_id worker: {str(e)}") + logger.error(traceback.format_exc()) + raise + finally: + __job_sleep() From c0e2378e95292f2169b73afe462d435a6f08fff9 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 00:49:48 -0800 Subject: [PATCH 08/32] first draft for dead letter queue --- PixivServer/app.py | 6 + PixivServer/config/celery.py | 15 +- PixivServer/config/rabbitmq.py | 1 + PixivServer/routers/dlq.py | 171 ++++++++++++++++++ PixivServer/worker/__init__.py | 9 +- .../pixivutil_server_common/models.py | 6 + 6 files changed, 204 insertions(+), 4 deletions(-) create mode 100644 PixivServer/routers/dlq.py diff --git a/PixivServer/app.py b/PixivServer/app.py index a49ca79..49f1550 100644 --- a/PixivServer/app.py +++ b/PixivServer/app.py @@ -10,6 +10,7 @@ import PixivServer.auth import PixivServer.routers import PixivServer.routers.database +import PixivServer.routers.dlq import PixivServer.routers.download_queue import PixivServer.routers.health import PixivServer.routers.metadata_queue @@ -117,6 +118,11 @@ async def request_metrics_middleware(request: Request, call_next): prefix="/api/database", dependencies=auth_dependency, ) +app.include_router( + PixivServer.routers.dlq.router, + prefix="/api/queue/dead-letter", + dependencies=auth_dependency, +) # app.include_router( # PixivServer.routers.subscription.router, # prefix="/api/subscription" diff --git a/PixivServer/config/celery.py b/PixivServer/config/celery.py index a950fa0..be6806f 100644 --- a/PixivServer/config/celery.py +++ b/PixivServer/config/celery.py @@ -3,9 +3,22 @@ from PixivServer.config import rabbitmq default_exchange = Exchange('pixivutil-exchange', type='direct', durable=True, delivery_mode=2) +dlx_exchange = Exchange('pixivutil-dlx', type='fanout', durable=True, delivery_mode=2) +dead_letter_queue = Queue( + name="pixivutil-dead-letter", + exchange=dlx_exchange, + routing_key='', + durable=True, +) CELERY_QUEUES = ( - Queue(name="pixivutil-queue", exchange=default_exchange, routing_key='pixivutil-queue', durable=True), + Queue( + name="pixivutil-queue", + exchange=default_exchange, + routing_key='pixivutil-queue', + durable=True, + queue_arguments={'x-dead-letter-exchange': 'pixivutil-dlx'}, + ), ) BROKER_URL = rabbitmq.config.broker_url diff --git a/PixivServer/config/rabbitmq.py b/PixivServer/config/rabbitmq.py index c86d3e9..6525fda 100644 --- a/PixivServer/config/rabbitmq.py +++ b/PixivServer/config/rabbitmq.py @@ -5,5 +5,6 @@ class RabbitConfig: def __init__(self): self.broker_url = os.getenv("RABBITMQ_BROKER_URL", "amqp://guest:guest@rabbitmq:5672") + self.management_url = os.getenv("RABBITMQ_MANAGEMENT_URL", "http://guest:guest@rabbitmq:15672") config = RabbitConfig() diff --git a/PixivServer/routers/dlq.py b/PixivServer/routers/dlq.py new file mode 100644 index 0000000..8446eb6 --- /dev/null +++ b/PixivServer/routers/dlq.py @@ -0,0 +1,171 @@ +import asyncio +import base64 +import contextlib +import json +import logging +import urllib.error +import urllib.parse +import urllib.request + +from fastapi import APIRouter, HTTPException, Response +from fastapi.responses import JSONResponse +from kombu import Connection + +from PixivServer.config import rabbitmq +from PixivServer.config.celery import dead_letter_queue +from PixivServer.worker.download import ( + delete_artwork_by_id, + download_artworks_by_id, + download_artworks_by_member_id, + download_artworks_by_tag, +) +from PixivServer.worker.metadata import ( + download_artwork_metadata_by_id, + download_member_metadata_by_id, + download_series_metadata_by_id, + download_tag_metadata_by_id, +) + +logger = logging.getLogger('uvicorn.pixivutil') +router = APIRouter() + +_TASK_REGISTRY: dict = { + "download_artworks_by_id": download_artworks_by_id, + "download_artworks_by_member_id": download_artworks_by_member_id, + "download_artworks_by_tag": download_artworks_by_tag, + "delete_artwork_by_id": delete_artwork_by_id, + "download_artwork_metadata_by_id": download_artwork_metadata_by_id, + "download_member_metadata_by_id": download_member_metadata_by_id, + "download_series_metadata_by_id": download_series_metadata_by_id, + "download_tag_metadata_by_id": download_tag_metadata_by_id, +} + + +def _mgmt_get_messages(count: int = 100) -> list[dict]: + parsed = urllib.parse.urlparse(rabbitmq.config.management_url) + credentials = f"{parsed.username}:{parsed.password}" + auth = base64.b64encode(credentials.encode()).decode() + base_url = f"{parsed.scheme}://{parsed.hostname}:{parsed.port}" + url = f"{base_url}/api/queues/%2F/pixivutil-dead-letter/get" + data = json.dumps({"count": count, "ackmode": "ack_requeue_true", "encoding": "auto"}).encode() + req = urllib.request.Request(url, data=data, method="POST") + req.add_header("Content-Type", "application/json") + req.add_header("Authorization", f"Basic {auth}") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + +def _drain(conn) -> list: + bound = dead_letter_queue.bind(conn) + bound.declare() + msgs = [] + while True: + msg = bound.get(no_ack=False) + if msg is None: + break + msgs.append(msg) + return msgs + + +@router.get("/") +async def list_dead_letter_messages() -> Response: + """ + List all messages currently in the dead letter queue. + """ + try: + raw = await asyncio.to_thread(_mgmt_get_messages) + except (urllib.error.URLError, json.JSONDecodeError) as e: + raise HTTPException(status_code=503, detail=f"RabbitMQ management API unavailable: {e}") + messages = [] + for item in raw: + with contextlib.suppress(KeyError, json.JSONDecodeError): + messages.append(json.loads(item["payload"])) + return JSONResponse(messages) + + +@router.post("/resume") +async def resume_all_dead_letter_messages() -> Response: + """ + Requeue all dead letter messages to the main queue. + Messages with unrecognised task names are left in the dead letter queue. + """ + def _run() -> int: + count = 0 + with Connection(rabbitmq.config.broker_url) as conn: + for msg in _drain(conn): + body = msg.payload + task_fn = _TASK_REGISTRY.get(body.get("task_name")) + if task_fn is not None: + task_fn.delay(body.get("payload", {})) + msg.ack() + count += 1 + else: + logger.warning(f"Unknown task name in DLQ message, leaving in queue: {body.get('task_name')}") + msg.reject(requeue=True) + return count + + count = await asyncio.to_thread(_run) + return JSONResponse({"requeued": count}) + + +@router.post("/{dead_letter_id}/resume") +async def resume_dead_letter_message(dead_letter_id: str) -> Response: + """ + Requeue a specific dead letter message to the main queue by its dead_letter_id. + """ + def _run() -> str | None: + """Returns task_name on success, None if not found, 'unknown' if task unrecognised.""" + with Connection(rabbitmq.config.broker_url) as conn: + for msg in _drain(conn): + body = msg.payload + if body.get("dead_letter_id") == dead_letter_id: + task_fn = _TASK_REGISTRY.get(body.get("task_name")) + if task_fn is None: + msg.reject(requeue=True) + return "unknown" + task_fn.delay(body.get("payload", {})) + msg.ack() + return body.get("task_name") + msg.reject(requeue=True) + return None + + result = await asyncio.to_thread(_run) + if result is None: + raise HTTPException(status_code=404, detail=f"Dead letter message not found: {dead_letter_id}") + if result == "unknown": + raise HTTPException(status_code=422, detail=f"Task name not recognised for dead letter message: {dead_letter_id}") + return JSONResponse({"dead_letter_id": dead_letter_id, "requeued": True, "task_name": result}) + + +@router.delete("/") +async def drop_all_dead_letter_messages() -> Response: + """ + Purge all messages from the dead letter queue. + """ + def _run() -> int: + with Connection(rabbitmq.config.broker_url) as conn: + return dead_letter_queue.bind(conn).purge() + + count = await asyncio.to_thread(_run) + return JSONResponse({"dropped": count}) + + +@router.delete("/{dead_letter_id}") +async def drop_dead_letter_message(dead_letter_id: str) -> Response: + """ + Drop a specific dead letter message by its dead_letter_id. + """ + def _run() -> bool: + with Connection(rabbitmq.config.broker_url) as conn: + for msg in _drain(conn): + body = msg.payload + if body.get("dead_letter_id") == dead_letter_id: + msg.ack() + return True + msg.reject(requeue=True) + return False + + found = await asyncio.to_thread(_run) + if not found: + raise HTTPException(status_code=404, detail=f"Dead letter message not found: {dead_letter_id}") + return JSONResponse({"dead_letter_id": dead_letter_id, "dropped": True}) diff --git a/PixivServer/worker/__init__.py b/PixivServer/worker/__init__.py index 98a4c33..94c1bf1 100644 --- a/PixivServer/worker/__init__.py +++ b/PixivServer/worker/__init__.py @@ -6,6 +6,7 @@ import PixivServer import PixivServer.service import PixivServer.service.pixiv +from PixivServer.config.celery import dead_letter_queue from PixivServer.config.server import config as server_config logger = logging.getLogger(__name__) @@ -15,9 +16,10 @@ @worker_init.connect -def on_worker_init(*args, **kwargs): +def on_worker_init(sender, **kwargs): + with sender.app.connection() as conn: + dead_letter_queue.bind(conn).declare() PixivServer.service.pixiv.service.open() - return @worker_shutdown.connect @@ -53,7 +55,8 @@ def config_loggers(*args, **kwargs): # logger.info('Running scheduled tag subscription job...') # subscription_service.run_tag_subscription_job() -# Register task modules +# Register task modules, as @shared_task decorator only runs when the module is imported. +# until then, the task functions don't exist in Celery's registry. import PixivServer.worker.download # noqa: E402, F401 import PixivServer.worker.metadata # noqa: E402, F401 diff --git a/PixivServerCommon/pixivutil_server_common/models.py b/PixivServerCommon/pixivutil_server_common/models.py index a0b1f41..1db8d56 100644 --- a/PixivServerCommon/pixivutil_server_common/models.py +++ b/PixivServerCommon/pixivutil_server_common/models.py @@ -5,6 +5,12 @@ from pydantic import BaseModel, Field +class DeadLetterMessage(BaseModel): + dead_letter_id: str + task_name: str + payload: dict + + class QueueTaskResponse(BaseModel): task_id: str artwork_id: str | int | None = None From d8af72ee73e0332327ec95b482550a927184035d Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 00:55:11 -0800 Subject: [PATCH 09/32] add tests for celery config --- tests/test_celery_config.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/test_celery_config.py diff --git a/tests/test_celery_config.py b/tests/test_celery_config.py new file mode 100644 index 0000000..d739db0 --- /dev/null +++ b/tests/test_celery_config.py @@ -0,0 +1,20 @@ +from celery import Celery + + +def test_celery_failure_is_rejected_for_rabbitmq_dlq(): + app = Celery("pixivutil-test") + app.config_from_object("PixivServer.config.celery") + + assert app.conf.task_acks_late is True + assert app.conf.task_acks_on_failure_or_timeout is False + + +def test_main_queue_declares_dead_letter_exchange(): + app = Celery("pixivutil-test") + app.config_from_object("PixivServer.config.celery") + + queues = {queue.name: queue for queue in app.conf.CELERY_QUEUES} + assert "pixivutil-queue" in queues + assert queues["pixivutil-queue"].queue_arguments == { + "x-dead-letter-exchange": "pixivutil-dlx", + } From ea98fe8729b3229f140a5aefe384e21fa0da17ba Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 01:18:06 -0800 Subject: [PATCH 10/32] fixes --- PixivServer/config/celery.py | 1 + PixivServer/routers/dlq.py | 228 +++++++++++++++++++++++++---------- PixivServer/worker/dev.py | 45 +++---- 3 files changed, 190 insertions(+), 84 deletions(-) diff --git a/PixivServer/config/celery.py b/PixivServer/config/celery.py index be6806f..36bcafa 100644 --- a/PixivServer/config/celery.py +++ b/PixivServer/config/celery.py @@ -24,4 +24,5 @@ BROKER_URL = rabbitmq.config.broker_url CELERY_ACKS_LATE = True CELERY_TASK_ACKS_LATE = True +CELERY_ACKS_ON_FAILURE_OR_TIMEOUT = False CELERY_TASK_REJECT_ON_WORKER_LOST = True diff --git a/PixivServer/routers/dlq.py b/PixivServer/routers/dlq.py index 8446eb6..6197b36 100644 --- a/PixivServer/routers/dlq.py +++ b/PixivServer/routers/dlq.py @@ -1,58 +1,60 @@ import asyncio -import base64 -import contextlib -import json import logging -import urllib.error -import urllib.parse -import urllib.request +from typing import Any from fastapi import APIRouter, HTTPException, Response from fastapi.responses import JSONResponse from kombu import Connection from PixivServer.config import rabbitmq -from PixivServer.config.celery import dead_letter_queue -from PixivServer.worker.download import ( - delete_artwork_by_id, - download_artworks_by_id, - download_artworks_by_member_id, - download_artworks_by_tag, -) -from PixivServer.worker.metadata import ( - download_artwork_metadata_by_id, - download_member_metadata_by_id, - download_series_metadata_by_id, - download_tag_metadata_by_id, -) +from PixivServer.config.celery import dead_letter_queue, default_exchange +from PixivServer.worker import pixiv_worker logger = logging.getLogger('uvicorn.pixivutil') router = APIRouter() -_TASK_REGISTRY: dict = { - "download_artworks_by_id": download_artworks_by_id, - "download_artworks_by_member_id": download_artworks_by_member_id, - "download_artworks_by_tag": download_artworks_by_tag, - "delete_artwork_by_id": delete_artwork_by_id, - "download_artwork_metadata_by_id": download_artwork_metadata_by_id, - "download_member_metadata_by_id": download_member_metadata_by_id, - "download_series_metadata_by_id": download_series_metadata_by_id, - "download_tag_metadata_by_id": download_tag_metadata_by_id, -} - - -def _mgmt_get_messages(count: int = 100) -> list[dict]: - parsed = urllib.parse.urlparse(rabbitmq.config.management_url) - credentials = f"{parsed.username}:{parsed.password}" - auth = base64.b64encode(credentials.encode()).decode() - base_url = f"{parsed.scheme}://{parsed.hostname}:{parsed.port}" - url = f"{base_url}/api/queues/%2F/pixivutil-dead-letter/get" - data = json.dumps({"count": count, "ackmode": "ack_requeue_true", "encoding": "auto"}).encode() - req = urllib.request.Request(url, data=data, method="POST") - req.add_header("Content-Type", "application/json") - req.add_header("Authorization", f"Basic {auth}") - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read()) + +def _extract_task_payload_from_celery_body(body: Any) -> dict: + # Celery protocol v2 JSON body is typically [args, kwargs, embed]. + if not isinstance(body, list) or len(body) < 2: + return {} + args = body[0] + kwargs = body[1] + if isinstance(args, list) and len(args) == 1 and isinstance(args[0], dict): + return args[0] + if isinstance(kwargs, dict) and "request_dict" in kwargs and isinstance(kwargs["request_dict"], dict): + return kwargs["request_dict"] + if isinstance(kwargs, dict): + return kwargs + return {} + + +def _normalize_dead_letter_payload(body: Any, headers: dict | None = None) -> dict | None: + if isinstance(body, dict): + if "dead_letter_id" in body and "task_name" in body: + return { + "dead_letter_id": str(body["dead_letter_id"]), + "task_name": str(body["task_name"]), + "payload": body.get("payload", {}) if isinstance(body.get("payload", {}), dict) else {}, + } + if "task_name" in body and "payload" in body: + # Backfill a stable identifier for older custom format if present. + return { + "dead_letter_id": str(body.get("dead_letter_id") or body.get("task_id") or ""), + "task_name": str(body["task_name"]), + "payload": body.get("payload", {}) if isinstance(body.get("payload", {}), dict) else {}, + } + + headers = headers or {} + task_name = headers.get("task") + task_id = headers.get("id") or headers.get("task_id") + if isinstance(task_name, str): + return { + "dead_letter_id": str(task_id or ""), + "task_name": task_name, + "payload": _extract_task_payload_from_celery_body(body), + } + return None def _drain(conn) -> list: @@ -67,20 +69,114 @@ def _drain(conn) -> list: return msgs +def _kombu_message_to_dlq_record(msg) -> dict | None: + headers = msg.headers if isinstance(msg.headers, dict) else {} + return _normalize_dead_letter_payload(msg.payload, headers=headers) + + +def _get_registered_task(task_name: str | None): + if not task_name: + return None + # Skip Celery internals/builtins even if registered. + if task_name.startswith("celery."): + return None + task = pixiv_worker.tasks.get(task_name) + return task if task is not None else None + + +def _is_native_celery_message(msg) -> bool: + headers = msg.headers if isinstance(msg.headers, dict) else {} + return isinstance(headers.get("task"), str) + + +def _clean_republish_headers(headers: dict) -> dict: + # Drop broker-added dead-letter metadata so replayed messages don't keep stale x-death history. + ignored = { + "x-death", + "x-first-death-exchange", + "x-first-death-queue", + "x-first-death-reason", + "x-last-death-exchange", + "x-last-death-queue", + "x-last-death-reason", + } + return {k: v for k, v in headers.items() if k not in ignored} + + +def _republish_native_celery_message(conn, msg) -> str | None: + if not _is_native_celery_message(msg): + return None + + headers = msg.headers if isinstance(msg.headers, dict) else {} + task_name = headers.get("task") + if not isinstance(task_name, str): + return None + + props = msg.properties if isinstance(msg.properties, dict) else {} + publish_props = {} + for key in ( + "correlation_id", + "reply_to", + "priority", + "expiration", + "message_id", + "timestamp", + "type", + "app_id", + ): + value = props.get(key) + if value is not None: + publish_props[key] = value + + with conn.Producer() as producer: + raw_body = msg.body + if isinstance(raw_body, str): + raw_body = raw_body.encode(msg.content_encoding or "utf-8") + if isinstance(raw_body, memoryview): + raw_body = raw_body.tobytes() + producer.publish( + raw_body, + exchange=default_exchange, + routing_key="pixivutil-queue", + headers=_clean_republish_headers(headers), + content_type=msg.content_type, + content_encoding=msg.content_encoding, + delivery_mode=2, + **publish_props, + ) + return task_name + + +def _resume_message(conn, msg, body: dict) -> str | None: + task_name = _republish_native_celery_message(conn, msg) + if task_name is not None: + return task_name + + task_fn = _get_registered_task(body.get("task_name")) + if task_fn is None: + return None + task_fn.delay(body.get("payload", {})) + return body.get("task_name") + + @router.get("/") async def list_dead_letter_messages() -> Response: """ List all messages currently in the dead letter queue. """ - try: - raw = await asyncio.to_thread(_mgmt_get_messages) - except (urllib.error.URLError, json.JSONDecodeError) as e: - raise HTTPException(status_code=503, detail=f"RabbitMQ management API unavailable: {e}") - messages = [] - for item in raw: - with contextlib.suppress(KeyError, json.JSONDecodeError): - messages.append(json.loads(item["payload"])) - return JSONResponse(messages) + def _run() -> list[dict]: + messages: list[dict] = [] + with Connection(rabbitmq.config.broker_url) as conn: + for msg in _drain(conn): + try: + normalized = _kombu_message_to_dlq_record(msg) + if normalized is not None: + messages.append(normalized) + finally: + msg.reject(requeue=True) + return messages + + return JSONResponse(await asyncio.to_thread(_run)) @router.post("/resume") @@ -93,10 +189,13 @@ def _run() -> int: count = 0 with Connection(rabbitmq.config.broker_url) as conn: for msg in _drain(conn): - body = msg.payload - task_fn = _TASK_REGISTRY.get(body.get("task_name")) - if task_fn is not None: - task_fn.delay(body.get("payload", {})) + body = _kombu_message_to_dlq_record(msg) + if body is None: + logger.warning("Unparseable DLQ message, leaving in queue") + msg.reject(requeue=True) + continue + resumed_task_name = _resume_message(conn, msg, body) + if resumed_task_name is not None: msg.ack() count += 1 else: @@ -117,15 +216,17 @@ def _run() -> str | None: """Returns task_name on success, None if not found, 'unknown' if task unrecognised.""" with Connection(rabbitmq.config.broker_url) as conn: for msg in _drain(conn): - body = msg.payload + body = _kombu_message_to_dlq_record(msg) + if body is None: + msg.reject(requeue=True) + continue if body.get("dead_letter_id") == dead_letter_id: - task_fn = _TASK_REGISTRY.get(body.get("task_name")) - if task_fn is None: + resumed_task_name = _resume_message(conn, msg, body) + if resumed_task_name is None: msg.reject(requeue=True) return "unknown" - task_fn.delay(body.get("payload", {})) msg.ack() - return body.get("task_name") + return resumed_task_name msg.reject(requeue=True) return None @@ -158,7 +259,10 @@ async def drop_dead_letter_message(dead_letter_id: str) -> Response: def _run() -> bool: with Connection(rabbitmq.config.broker_url) as conn: for msg in _drain(conn): - body = msg.payload + body = _kombu_message_to_dlq_record(msg) + if body is None: + msg.reject(requeue=True) + continue if body.get("dead_letter_id") == dead_letter_id: msg.ack() return True diff --git a/PixivServer/worker/dev.py b/PixivServer/worker/dev.py index a9e482f..4bf0218 100644 --- a/PixivServer/worker/dev.py +++ b/PixivServer/worker/dev.py @@ -1,32 +1,33 @@ import logging -import random -import time -import traceback +from pathlib import Path from celery import shared_task -import PixivServer.service.pixiv from PixivServer.models.pixiv_worker import DownloadArtworkByIdRequest logger = logging.getLogger(__name__) +_SIMULATED_RETRY_COUNTDOWN = 1 +_SIMULATED_MAX_RETRIES = 1 +_SUCCESS_SENTINEL_PATH = Path("/tmp/pixivutil-dev-dlq-success.flag") -def __job_sleep(): - time.sleep(random.uniform(1, 5)) - return 0 - - -@shared_task(name="dev_download_artworks_by_id", queue='pixivutil-queue') -def dev_download_artworks_by_id(request_dict: dict): - try: - request = DownloadArtworkByIdRequest(**request_dict) - PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"(test) Downloading artwork by ID: {request.artwork_id}.") - time.sleep(3) - PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"(test) Completed download artwork by ID: {request.artwork_id}.") +# This endpoint exists to confirm DLQ functionality. +# Use this for any kind of DLQ-related task and make any necessary changes in logic to prove/confirm hypotheses. +# This will be commented out once DLQ is stable, so in the meantime do whatever you want with this endpoint, +# just clean it up when done and don't commit things back in. +@shared_task(bind=True, name="dev_download_artworks_by_id", queue='pixivutil-queue') +def dev_download_artworks_by_id(self, request_dict: dict): + request = DownloadArtworkByIdRequest(**request_dict) + attempt = self.request.retries + 1 + max_attempts = _SIMULATED_MAX_RETRIES + 1 + logger.error(f"(dev) Attempt {attempt}/{max_attempts} for artwork_id={request.artwork_id}") + if attempt < max_attempts: + raise self.retry( + exc=ConnectionError("Simulated network failure"), + countdown=_SIMULATED_RETRY_COUNTDOWN, + ) + if _SUCCESS_SENTINEL_PATH.exists(): + logger.error(f"(dev) Sentinel found at {_SUCCESS_SENTINEL_PATH}; succeeding on resumed run") return True - except Exception as e: - logger.error(f"Error in download_artworks_by_id worker: {str(e)}") - logger.error(traceback.format_exc()) - raise - finally: - __job_sleep() + logger.error(f"(dev) Max retries exceeded for artwork_id={request.artwork_id}, raising terminal failure for broker DLQ") + raise ConnectionError("Simulated terminal failure after retries") From e37b4f8a82b7389a49b4eea829d8bc938a3cee39 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 01:47:50 -0800 Subject: [PATCH 11/32] add integration tests --- integration_tests/README.md | 9 ++ integration_tests/__init__.py | 1 + integration_tests/conftest.py | 156 +++++++++++++++++++++++++++ integration_tests/test_dlq_replay.py | 54 ++++++++++ pyproject.toml | 2 +- 5 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 integration_tests/README.md create mode 100644 integration_tests/__init__.py create mode 100644 integration_tests/conftest.py create mode 100644 integration_tests/test_dlq_replay.py diff --git a/integration_tests/README.md b/integration_tests/README.md new file mode 100644 index 0000000..92d2beb --- /dev/null +++ b/integration_tests/README.md @@ -0,0 +1,9 @@ +# PixivUtil Server Integration Test Suite + +Run integration tests: + +```sh +uv run pytest integration_tests +``` + +Docker and Docker Compose are required. diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/integration_tests/__init__.py @@ -0,0 +1 @@ + diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py new file mode 100644 index 0000000..46b40c9 --- /dev/null +++ b/integration_tests/conftest.py @@ -0,0 +1,156 @@ +import json +import shutil +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +BASE_URL = "http://localhost:8000" +AUTH_HEADER = ("Authorization", "Bearer pixiv") +DEV_SUCCESS_SENTINEL = "/tmp/pixivutil-dev-dlq-success.flag" # TODO: this requires a redesign. + + +def _run( + args: list[str], + *, + check: bool = True, + timeout: int = 120, + cwd: Path = REPO_ROOT, +) -> subprocess.CompletedProcess[str]: + try: + proc = subprocess.run( + args, + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except FileNotFoundError as exc: + raise RuntimeError(f"Command not found: {args[0]}") from exc + except subprocess.TimeoutExpired as exc: + raise RuntimeError(f"Command timed out ({timeout}s): {' '.join(args)}") from exc + + if check and proc.returncode != 0: + stderr = proc.stderr.strip() + stdout = proc.stdout.strip() + detail = stderr or stdout + raise RuntimeError(f"Command failed ({proc.returncode}): {' '.join(args)}\n{detail}") + return proc + + +def _require_docker_tools() -> None: + if shutil.which("docker") is None: + pytest.exit("docker is not available in PATH", returncode=1) + _run(["docker", "--version"], timeout=15) + _run(["docker", "compose", "version"], timeout=15) + + +def _http_json(method: str, path: str) -> object: + req = urllib.request.Request(f"{BASE_URL}{path}", method=method) + req.add_header(*AUTH_HEADER) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read()) + + +class ComposeTestEnv: + def compose(self, *args: str, check: bool = True, timeout: int = 240) -> subprocess.CompletedProcess[str]: + return _run(["docker", "compose", *args], check=check, timeout=timeout) + + def docker_exec(self, container: str, *args: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess[str]: + return _run(["docker", "exec", container, *args], check=check, timeout=timeout) + + def wait_http_ready(self, timeout: int = 60) -> None: + deadline = time.time() + timeout + last_error: str | None = None + while time.time() < deadline: + try: + req = urllib.request.Request(f"{BASE_URL}/", method="GET") + with urllib.request.urlopen(req, timeout=2): + return + except Exception as exc: # noqa: BLE001 + last_error = str(exc) + time.sleep(1) + raise RuntimeError(f"Server did not become ready within {timeout}s: {last_error}") + + def api_json(self, method: str, path: str) -> object: + return _http_json(method, path) + + def rabbitmq_queue_counts(self) -> dict[str, int]: + proc = self.docker_exec("rabbitmq", "rabbitmqctl", "list_queues", "name", "messages", timeout=90) + counts: dict[str, int] = {} + for line in proc.stdout.splitlines(): + if "\t" not in line: + continue + name, count = line.split("\t", 1) + if name == "name": + continue + try: + counts[name.strip()] = int(count.strip()) + except ValueError: + continue + return counts + + def wait_for_queue_count(self, queue_name: str, expected: int, timeout: int = 45) -> None: + deadline = time.time() + timeout + last_count: int | None = None + while time.time() < deadline: + last_count = self.rabbitmq_queue_counts().get(queue_name) + if last_count == expected: + return + time.sleep(1) + raise RuntimeError( + f"Queue {queue_name!r} did not reach {expected} within {timeout}s (last={last_count})" + ) + + def wait_worker_log_contains(self, needle: str, timeout: int = 45) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + logs = self.compose("logs", "--tail=400", "pixivutil-worker", timeout=90).stdout + if needle in logs: + return + time.sleep(1) + raise RuntimeError(f"Worker logs did not contain expected text within {timeout}s: {needle}") + + def clear_state(self) -> None: + self.docker_exec( + "pixivutil-worker", + "sh", + "-lc", + f"rm -f {DEV_SUCCESS_SENTINEL}", + check=False, + ) + self.docker_exec( + "rabbitmq", + "rabbitmqctl", + "purge_queue", + "pixivutil-dead-letter", + check=False, + timeout=90, + ) + + +def pytest_sessionstart(session: pytest.Session) -> None: + _require_docker_tools() + + +@pytest.fixture(scope="session") +def compose_env() -> ComposeTestEnv: + env = ComposeTestEnv() + env.compose("down", "--volumes", check=False, timeout=180) + env.compose("up", "--build", "-d", timeout=600) + env.wait_http_ready() + yield env + env.compose("down", "--volumes", check=False, timeout=180) + + +@pytest.fixture +def clean_env(compose_env: ComposeTestEnv) -> ComposeTestEnv: + compose_env.clear_state() + compose_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=30) + yield compose_env + compose_env.clear_state() diff --git a/integration_tests/test_dlq_replay.py b/integration_tests/test_dlq_replay.py new file mode 100644 index 0000000..25be39c --- /dev/null +++ b/integration_tests/test_dlq_replay.py @@ -0,0 +1,54 @@ +def test_dlq_resume_replays_native_celery_message(clean_env): + task = clean_env.api_json("POST", "/api/dev/artwork/424242") + task_id = task["task_id"] + + clean_env.wait_for_queue_count("pixivutil-dead-letter", 1, timeout=60) + messages = clean_env.api_json("GET", "/api/queue/dead-letter/") + assert len(messages) == 1 + message = messages[0] + assert message["dead_letter_id"] == task_id + assert message["task_name"] == "dev_download_artworks_by_id" + assert message["payload"]["artwork_id"] == 424242 + + clean_env.docker_exec( + "pixivutil-worker", + "sh", + "-lc", + "touch /tmp/pixivutil-dev-dlq-success.flag", + ) + + resumed = clean_env.api_json("POST", f"/api/queue/dead-letter/{task_id}/resume") + assert resumed["requeued"] is True + assert resumed["task_name"] == "dev_download_artworks_by_id" + + clean_env.wait_worker_log_contains("Sentinel found at /tmp/pixivutil-dev-dlq-success.flag; succeeding on resumed run") + clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=60) + assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] + + +def test_dlq_resume_all_replays_multiple_native_celery_messages(clean_env): + task_a = clean_env.api_json("POST", "/api/dev/artwork/111111") + task_b = clean_env.api_json("POST", "/api/dev/artwork/222222") + + clean_env.wait_for_queue_count("pixivutil-dead-letter", 2, timeout=90) + messages = clean_env.api_json("GET", "/api/queue/dead-letter/") + assert len(messages) == 2 + + ids = {message["dead_letter_id"] for message in messages} + assert ids == {task_a["task_id"], task_b["task_id"]} + assert {message["payload"]["artwork_id"] for message in messages} == {111111, 222222} + + clean_env.docker_exec( + "pixivutil-worker", + "sh", + "-lc", + "touch /tmp/pixivutil-dev-dlq-success.flag", + ) + + resumed = clean_env.api_json("POST", "/api/queue/dead-letter/resume") + assert resumed["requeued"] == 2 + + clean_env.wait_worker_log_contains("Sentinel found at /tmp/pixivutil-dev-dlq-success.flag; succeeding on resumed run") + clean_env.wait_for_queue_count("pixivutil-queue", 0, timeout=90) + clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=90) + assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] diff --git a/pyproject.toml b/pyproject.toml index e7eeb89..0cb29a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ pixivutil2 = [ [tool.pyright] pythonVersion = "3.12" -include = ["PixivServer", "PixivServerCommon", "PixivUtilClient", "tests"] +include = ["PixivServer", "PixivServerCommon", "PixivUtilClient", "tests", "integration_tests"] exclude = [ "PixivUtil2", "**/node_modules", From 67f82aa0aca092ccc7fc1b33af8fee98a88ae45c Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 01:54:58 -0800 Subject: [PATCH 12/32] add pixiv-api flag to integration tests --- integration_tests/conftest.py | 30 ++++++++++++++++++++++++---- integration_tests/test_dlq_replay.py | 5 +++++ tests/conftest.py | 25 +++++++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 46b40c9..6b2fa7a 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -14,6 +14,31 @@ DEV_SUCCESS_SENTINEL = "/tmp/pixivutil-dev-dlq-success.flag" # TODO: this requires a redesign. +def pytest_addoption(parser: pytest.Parser): + parser.addoption( + "--pixiv-api", + action="store_true", + default=False, + help="Run tests that make Pixiv API calls.", + ) + + +def pytest_configure(config: pytest.Config): + config.addinivalue_line( + "markers", + "pixiv_api: test performs Pixiv API calls and is skipped unless --pixiv-api is provided.", + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]): + if config.getoption("--pixiv-api"): + return + skip_pixiv_api = pytest.mark.skip(reason="need --pixiv-api option enabled") + for item in items: + if "pixiv_api" in item.keywords: + item.add_marker(skip_pixiv_api) + + def _run( args: list[str], *, @@ -134,12 +159,9 @@ def clear_state(self) -> None: ) -def pytest_sessionstart(session: pytest.Session) -> None: - _require_docker_tools() - - @pytest.fixture(scope="session") def compose_env() -> ComposeTestEnv: + _require_docker_tools() env = ComposeTestEnv() env.compose("down", "--volumes", check=False, timeout=180) env.compose("up", "--build", "-d", timeout=600) diff --git a/integration_tests/test_dlq_replay.py b/integration_tests/test_dlq_replay.py index 25be39c..eff9c2a 100644 --- a/integration_tests/test_dlq_replay.py +++ b/integration_tests/test_dlq_replay.py @@ -1,3 +1,7 @@ +import pytest + + +@pytest.mark.pixiv_api def test_dlq_resume_replays_native_celery_message(clean_env): task = clean_env.api_json("POST", "/api/dev/artwork/424242") task_id = task["task_id"] @@ -26,6 +30,7 @@ def test_dlq_resume_replays_native_celery_message(clean_env): assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] +@pytest.mark.pixiv_api def test_dlq_resume_all_replays_multiple_native_celery_messages(clean_env): task_a = clean_env.api_json("POST", "/api/dev/artwork/111111") task_b = clean_env.api_json("POST", "/api/dev/artwork/222222") diff --git a/tests/conftest.py b/tests/conftest.py index a601177..268024a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,31 @@ import pytest +def pytest_addoption(parser: pytest.Parser): + parser.addoption( + "--pixiv-api", + action="store_true", + default=False, + help="Run tests that make Pixiv API calls.", + ) + + +def pytest_configure(config: pytest.Config): + config.addinivalue_line( + "markers", + "pixiv_api: test performs Pixiv API calls and is skipped unless --pixiv-api is provided.", + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]): + if config.getoption("--pixiv-api"): + return + skip_pixiv_api = pytest.mark.skip(reason="need --pixiv-api option enabled") + for item in items: + if "pixiv_api" in item.keywords: + item.add_marker(skip_pixiv_api) + + @pytest.fixture def temp_dir(): """ From 3f51000c4456a64ee6ec6aa81393f07426d89c98 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 14:26:55 -0800 Subject: [PATCH 13/32] add dlq client tooling --- PixivServer/routers/dev.py | 15 ++- PixivServer/routers/dlq.py | 6 +- PixivServer/worker/dev.py | 70 ++++++++++++++ .../pixivutil_server_common/models.py | 19 ++++ PixivUtilClient/pixivutil_client/client.py | 27 ++++++ PixivUtilClient/pixivutil_client/models.py | 10 ++ PixivUtilClient/tests/test_client.py | 59 ++++++++++++ conftest.py | 26 +++++ integration_tests/conftest.py | 63 ++++++------ integration_tests/test_dlq_replay.py | 96 ++++++++++++++++++- tests/conftest.py | 25 ----- 11 files changed, 358 insertions(+), 58 deletions(-) create mode 100644 conftest.py diff --git a/PixivServer/routers/dev.py b/PixivServer/routers/dev.py index dfc081f..c9894e2 100644 --- a/PixivServer/routers/dev.py +++ b/PixivServer/routers/dev.py @@ -1,11 +1,11 @@ import logging from celery.result import AsyncResult -from fastapi import APIRouter, Response +from fastapi import APIRouter, HTTPException, Response from fastapi.responses import JSONResponse from PixivServer.models.pixiv_worker import DownloadArtworkByIdRequest -from PixivServer.worker.dev import dev_download_artworks_by_id +from PixivServer.worker.dev import dev_download_artworks_by_id, get_dev_task_state logger = logging.getLogger('uvicorn.pixivutil') router = APIRouter() @@ -26,3 +26,14 @@ async def dev_queue_download_artwork_by_id(artwork_id: str) -> Response: "artwork_title": artwork_title, "member_name": member_name, }) + + +@router.get("/task/{task_id}") +async def dev_task_status(task_id: str) -> Response: + """ + (test) Return dev worker task attempt history and terminal status. + """ + state = get_dev_task_state(task_id) + if state is None: + raise HTTPException(status_code=404, detail=f"Dev task state not found: {task_id}") + return JSONResponse(state) diff --git a/PixivServer/routers/dlq.py b/PixivServer/routers/dlq.py index 6197b36..f36c4f0 100644 --- a/PixivServer/routers/dlq.py +++ b/PixivServer/routers/dlq.py @@ -90,7 +90,7 @@ def _is_native_celery_message(msg) -> bool: def _clean_republish_headers(headers: dict) -> dict: - # Drop broker-added dead-letter metadata so replayed messages don't keep stale x-death history. + # Drop broker-added dead-letter metadata and Celery execution state so replay acts like a fresh enqueue. ignored = { "x-death", "x-first-death-exchange", @@ -99,6 +99,9 @@ def _clean_republish_headers(headers: dict) -> dict: "x-last-death-exchange", "x-last-death-queue", "x-last-death-reason", + "retries", + "eta", + "expires", } return {k: v for k, v in headers.items() if k not in ignored} @@ -118,7 +121,6 @@ def _republish_native_celery_message(conn, msg) -> str | None: "correlation_id", "reply_to", "priority", - "expiration", "message_id", "timestamp", "type", diff --git a/PixivServer/worker/dev.py b/PixivServer/worker/dev.py index 4bf0218..e6da03e 100644 --- a/PixivServer/worker/dev.py +++ b/PixivServer/worker/dev.py @@ -1,5 +1,7 @@ +import json import logging from pathlib import Path +from typing import Any from celery import shared_task @@ -10,6 +12,70 @@ _SIMULATED_RETRY_COUNTDOWN = 1 _SIMULATED_MAX_RETRIES = 1 _SUCCESS_SENTINEL_PATH = Path("/tmp/pixivutil-dev-dlq-success.flag") +_TASK_STATE_PATH = Path("/workdir/.pixivUtil2/dev-dlq-task-state.json") + + +def _load_task_states() -> dict[str, dict[str, Any]]: + try: + data = json.loads(_TASK_STATE_PATH.read_text()) + except FileNotFoundError: + return {} + except json.JSONDecodeError: + logger.warning("(dev) Could not parse task state file; resetting") + return {} + if not isinstance(data, dict): + return {} + result: dict[str, dict[str, Any]] = {} + for key, value in data.items(): + if isinstance(key, str) and isinstance(value, dict): + result[key] = value + return result + + +def _save_task_states(states: dict[str, dict[str, Any]]) -> None: + _TASK_STATE_PATH.parent.mkdir(parents=True, exist_ok=True) + temp_path = _TASK_STATE_PATH.with_suffix(".tmp") + temp_path.write_text(json.dumps(states, sort_keys=True)) + temp_path.replace(_TASK_STATE_PATH) + + +def _record_task_attempt(task_id: str, artwork_id: int, attempt: int) -> None: + states = _load_task_states() + state = states.get(task_id) + if not isinstance(state, dict): + state = { + "task_id": task_id, + "artwork_id": artwork_id, + "attempt_history": [], + "terminal_state": None, + } + history = state.get("attempt_history") + if not isinstance(history, list): + history = [] + history.append(attempt) + state["attempt_history"] = history + state["task_id"] = task_id + state["artwork_id"] = artwork_id + state["last_attempt"] = attempt + state["terminal_state"] = None + states[task_id] = state + _save_task_states(states) + + +def _record_terminal_state(task_id: str, terminal_state: str) -> None: + states = _load_task_states() + state = states.get(task_id) + if not isinstance(state, dict): + state = {"task_id": task_id, "attempt_history": []} + state["task_id"] = task_id + state["terminal_state"] = terminal_state + states[task_id] = state + _save_task_states(states) + + +def get_dev_task_state(task_id: str) -> dict[str, Any] | None: + state = _load_task_states().get(task_id) + return state if isinstance(state, dict) else None # This endpoint exists to confirm DLQ functionality. # Use this for any kind of DLQ-related task and make any necessary changes in logic to prove/confirm hypotheses. @@ -18,8 +84,10 @@ @shared_task(bind=True, name="dev_download_artworks_by_id", queue='pixivutil-queue') def dev_download_artworks_by_id(self, request_dict: dict): request = DownloadArtworkByIdRequest(**request_dict) + task_id = str(self.request.id) attempt = self.request.retries + 1 max_attempts = _SIMULATED_MAX_RETRIES + 1 + _record_task_attempt(task_id, request.artwork_id, attempt) logger.error(f"(dev) Attempt {attempt}/{max_attempts} for artwork_id={request.artwork_id}") if attempt < max_attempts: raise self.retry( @@ -27,7 +95,9 @@ def dev_download_artworks_by_id(self, request_dict: dict): countdown=_SIMULATED_RETRY_COUNTDOWN, ) if _SUCCESS_SENTINEL_PATH.exists(): + _record_terminal_state(task_id, "succeeded") logger.error(f"(dev) Sentinel found at {_SUCCESS_SENTINEL_PATH}; succeeding on resumed run") return True + _record_terminal_state(task_id, "failed") logger.error(f"(dev) Max retries exceeded for artwork_id={request.artwork_id}, raising terminal failure for broker DLQ") raise ConnectionError("Simulated terminal failure after retries") diff --git a/PixivServerCommon/pixivutil_server_common/models.py b/PixivServerCommon/pixivutil_server_common/models.py index 1db8d56..c887d54 100644 --- a/PixivServerCommon/pixivutil_server_common/models.py +++ b/PixivServerCommon/pixivutil_server_common/models.py @@ -11,6 +11,25 @@ class DeadLetterMessage(BaseModel): payload: dict +class DeadLetterResumeAllResponse(BaseModel): + requeued: int + + +class DeadLetterResumeResponse(BaseModel): + dead_letter_id: str + requeued: bool + task_name: str + + +class DeadLetterDropAllResponse(BaseModel): + dropped: int + + +class DeadLetterDropResponse(BaseModel): + dead_letter_id: str + dropped: bool + + class QueueTaskResponse(BaseModel): task_id: str artwork_id: str | int | None = None diff --git a/PixivUtilClient/pixivutil_client/client.py b/PixivUtilClient/pixivutil_client/client.py index 233f1da..9edef4e 100644 --- a/PixivUtilClient/pixivutil_client/client.py +++ b/PixivUtilClient/pixivutil_client/client.py @@ -8,6 +8,11 @@ from pixivutil_client.exceptions import PixivAPIError, PixivTransportError from pixivutil_client.models import ( + DeadLetterDropAllResponse, + DeadLetterDropResponse, + DeadLetterMessage, + DeadLetterResumeAllResponse, + DeadLetterResumeResponse, PixivImageComplete, PixivMemberPortfolio, PixivSeriesInfo, @@ -243,3 +248,25 @@ async def reset_database(self) -> str: async def reset_downloads(self) -> str: payload = await self._request("DELETE", "/api/server/downloads") return str(payload) + + async def list_dead_letter_messages(self) -> list[DeadLetterMessage]: + payload = await self._request("GET", "/api/queue/dead-letter/") + return [DeadLetterMessage.model_validate(item) for item in payload] + + async def resume_all_dead_letter_messages(self) -> DeadLetterResumeAllResponse: + payload = await self._request("POST", "/api/queue/dead-letter/resume") + return DeadLetterResumeAllResponse.model_validate(payload) + + async def resume_dead_letter_message(self, dead_letter_id: str) -> DeadLetterResumeResponse: + encoded_dead_letter_id = quote(dead_letter_id, safe="") + payload = await self._request("POST", f"/api/queue/dead-letter/{encoded_dead_letter_id}/resume") + return DeadLetterResumeResponse.model_validate(payload) + + async def drop_all_dead_letter_messages(self) -> DeadLetterDropAllResponse: + payload = await self._request("DELETE", "/api/queue/dead-letter/") + return DeadLetterDropAllResponse.model_validate(payload) + + async def drop_dead_letter_message(self, dead_letter_id: str) -> DeadLetterDropResponse: + encoded_dead_letter_id = quote(dead_letter_id, safe="") + payload = await self._request("DELETE", f"/api/queue/dead-letter/{encoded_dead_letter_id}") + return DeadLetterDropResponse.model_validate(payload) diff --git a/PixivUtilClient/pixivutil_client/models.py b/PixivUtilClient/pixivutil_client/models.py index c710e1e..14571fc 100644 --- a/PixivUtilClient/pixivutil_client/models.py +++ b/PixivUtilClient/pixivutil_client/models.py @@ -3,6 +3,11 @@ """ from pixivutil_server_common.models import ( + DeadLetterDropAllResponse, + DeadLetterDropResponse, + DeadLetterMessage, + DeadLetterResumeAllResponse, + DeadLetterResumeResponse, PixivDateInfo, PixivImageComplete, PixivImageToSeries, @@ -23,6 +28,11 @@ ) __all__ = [ + "DeadLetterDropAllResponse", + "DeadLetterDropResponse", + "DeadLetterMessage", + "DeadLetterResumeAllResponse", + "DeadLetterResumeResponse", "PixivDateInfo", "PixivImageComplete", "PixivImageToSeries", diff --git a/PixivUtilClient/tests/test_client.py b/PixivUtilClient/tests/test_client.py index 85bcfe6..1187ec7 100644 --- a/PixivUtilClient/tests/test_client.py +++ b/PixivUtilClient/tests/test_client.py @@ -21,9 +21,45 @@ async def auth_echo(request: web.Request) -> web.Response: async def failure(_: web.Request) -> web.Response: return web.json_response({"error": "bad request"}, status=400) + async def dlq_list(_: web.Request) -> web.Response: + return web.json_response( + [ + { + "dead_letter_id": "abc-123", + "task_name": "download_artworks_by_id", + "payload": {"artwork_id": 42}, + } + ] + ) + + async def dlq_resume_all(_: web.Request) -> web.Response: + return web.json_response({"requeued": 2}) + + async def dlq_resume_one(request: web.Request) -> web.Response: + dead_letter_id = request.match_info["dead_letter_id"] + return web.json_response( + { + "dead_letter_id": dead_letter_id, + "requeued": True, + "task_name": "download_artworks_by_id", + } + ) + + async def dlq_drop_all(_: web.Request) -> web.Response: + return web.json_response({"dropped": 3}) + + async def dlq_drop_one(request: web.Request) -> web.Response: + dead_letter_id = request.match_info["dead_letter_id"] + return web.json_response({"dead_letter_id": dead_letter_id, "dropped": True}) + app.router.add_get("/api/database/members", plain_json) app.router.add_post("/api/queue/download/artwork/123", auth_echo) app.router.add_get("/boom", failure) + app.router.add_get("/api/queue/dead-letter/", dlq_list) + app.router.add_post("/api/queue/dead-letter/resume", dlq_resume_all) + app.router.add_post("/api/queue/dead-letter/{dead_letter_id}/resume", dlq_resume_one) + app.router.add_delete("/api/queue/dead-letter/", dlq_drop_all) + app.router.add_delete("/api/queue/dead-letter/{dead_letter_id}", dlq_drop_one) runner = web.AppRunner(app) await runner.setup() @@ -93,3 +129,26 @@ def request(self, method: str, url: str, **kwargs: Any) -> FakeContextManager: payload = await client._request("GET", "/probe") assert payload["ok"] is True assert captured["kwargs"]["ssl"] is False + + +@pytest.mark.asyncio +async def test_dead_letter_queue_client_methods(server_url: str) -> None: + async with PixivAsyncClient(server_url) as client: + messages = await client.list_dead_letter_messages() + assert len(messages) == 1 + assert messages[0].dead_letter_id == "abc-123" + assert messages[0].payload["artwork_id"] == 42 + + resumed_all = await client.resume_all_dead_letter_messages() + assert resumed_all.requeued == 2 + + resumed_one = await client.resume_dead_letter_message("abc-123") + assert resumed_one.dead_letter_id == "abc-123" + assert resumed_one.requeued is True + + dropped_all = await client.drop_all_dead_letter_messages() + assert dropped_all.dropped == 3 + + dropped_one = await client.drop_dead_letter_message("abc-123") + assert dropped_one.dead_letter_id == "abc-123" + assert dropped_one.dropped is True diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..96d8c54 --- /dev/null +++ b/conftest.py @@ -0,0 +1,26 @@ +import pytest + + +def pytest_addoption(parser: pytest.Parser): + parser.addoption( + "--pixiv-api", + action="store_true", + default=False, + help="Run tests that make Pixiv API calls.", + ) + + +def pytest_configure(config: pytest.Config): + config.addinivalue_line( + "markers", + "pixiv_api: test performs Pixiv API calls and is skipped unless --pixiv-api is provided.", + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]): + if config.getoption("--pixiv-api"): + return + skip_pixiv_api = pytest.mark.skip(reason="need --pixiv-api option enabled") + for item in items: + if "pixiv_api" in item.keywords: + item.add_marker(skip_pixiv_api) diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 6b2fa7a..19600d7 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -13,32 +13,6 @@ AUTH_HEADER = ("Authorization", "Bearer pixiv") DEV_SUCCESS_SENTINEL = "/tmp/pixivutil-dev-dlq-success.flag" # TODO: this requires a redesign. - -def pytest_addoption(parser: pytest.Parser): - parser.addoption( - "--pixiv-api", - action="store_true", - default=False, - help="Run tests that make Pixiv API calls.", - ) - - -def pytest_configure(config: pytest.Config): - config.addinivalue_line( - "markers", - "pixiv_api: test performs Pixiv API calls and is skipped unless --pixiv-api is provided.", - ) - - -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]): - if config.getoption("--pixiv-api"): - return - skip_pixiv_api = pytest.mark.skip(reason="need --pixiv-api option enabled") - for item in items: - if "pixiv_api" in item.keywords: - item.add_marker(skip_pixiv_api) - - def _run( args: list[str], *, @@ -105,6 +79,12 @@ def wait_http_ready(self, timeout: int = 60) -> None: def api_json(self, method: str, path: str) -> object: return _http_json(method, path) + def dev_task_state(self, task_id: str) -> dict: + state = self.api_json("GET", f"/api/dev/task/{task_id}") + if not isinstance(state, dict): + raise RuntimeError(f"Unexpected dev task state payload for {task_id}: {state!r}") + return state + def rabbitmq_queue_counts(self) -> dict[str, int]: proc = self.docker_exec("rabbitmq", "rabbitmqctl", "list_queues", "name", "messages", timeout=90) counts: dict[str, int] = {} @@ -141,13 +121,41 @@ def wait_worker_log_contains(self, needle: str, timeout: int = 45) -> None: time.sleep(1) raise RuntimeError(f"Worker logs did not contain expected text within {timeout}s: {needle}") + def wait_for_dev_task_terminal_state(self, task_id: str, terminal_state: str, timeout: int = 45) -> dict: + deadline = time.time() + timeout + last_state: dict | None = None + last_error: str | None = None + while time.time() < deadline: + try: + state = self.dev_task_state(task_id) + last_state = state + if state.get("terminal_state") == terminal_state: + return state + except urllib.error.HTTPError as exc: + last_error = str(exc) + if exc.code != 404: + raise + time.sleep(1) + raise RuntimeError( + f"Dev task {task_id!r} did not reach terminal_state={terminal_state!r} within {timeout}s " + f"(last_state={last_state}, last_error={last_error})" + ) + def clear_state(self) -> None: self.docker_exec( "pixivutil-worker", "sh", "-lc", - f"rm -f {DEV_SUCCESS_SENTINEL}", + f"rm -f {DEV_SUCCESS_SENTINEL} /workdir/.pixivUtil2/dev-dlq-task-state.json /workdir/.pixivUtil2/dev-dlq-task-state.tmp", + check=False, + ) + self.docker_exec( + "rabbitmq", + "rabbitmqctl", + "purge_queue", + "pixivutil-queue", check=False, + timeout=90, ) self.docker_exec( "rabbitmq", @@ -173,6 +181,7 @@ def compose_env() -> ComposeTestEnv: @pytest.fixture def clean_env(compose_env: ComposeTestEnv) -> ComposeTestEnv: compose_env.clear_state() + compose_env.wait_for_queue_count("pixivutil-queue", 0, timeout=30) compose_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=30) yield compose_env compose_env.clear_state() diff --git a/integration_tests/test_dlq_replay.py b/integration_tests/test_dlq_replay.py index eff9c2a..327f002 100644 --- a/integration_tests/test_dlq_replay.py +++ b/integration_tests/test_dlq_replay.py @@ -1,8 +1,21 @@ import pytest +def _attempt_history(state: dict) -> list[int]: + history = state.get("attempt_history") + if not isinstance(history, list): + return [] + return [int(item) for item in history] + + @pytest.mark.pixiv_api def test_dlq_resume_replays_native_celery_message(clean_env): + """ + Replaying a single native Celery DLQ message should republish it back to the main queue and remove it from DLQ. + + This validates the baseline operator workflow for manual recovery using the dev endpoint and DLQ resume endpoint. + If this fails, administrators cannot reliably recover failed tasks from the dead letter queue. + """ task = clean_env.api_json("POST", "/api/dev/artwork/424242") task_id = task["task_id"] @@ -25,13 +38,55 @@ def test_dlq_resume_replays_native_celery_message(clean_env): assert resumed["requeued"] is True assert resumed["task_name"] == "dev_download_artworks_by_id" - clean_env.wait_worker_log_contains("Sentinel found at /tmp/pixivutil-dev-dlq-success.flag; succeeding on resumed run") + state = clean_env.wait_for_dev_task_terminal_state(task_id, "succeeded", timeout=90) + assert _attempt_history(state) == [1, 2, 1, 2] clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=60) assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] +@pytest.mark.pixiv_api +def test_dlq_resume_resets_retry_counter_for_replayed_native_celery_message(clean_env): + """ + Replaying a single native Celery message should reset retry state so the task starts a fresh retry lifecycle. + + DLQ replay is an operator recovery action, so the resumed task is expected to behave like a new enqueue rather than + inheriting an exhausted retry budget from the failed message. If this fails, replayed transient failures may skip + retry-attempt 1 and fail immediately or earlier than expected. + """ + artwork_id = 333333 + task = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_id}") + task_id = task["task_id"] + + clean_env.wait_for_queue_count("pixivutil-dead-letter", 1, timeout=60) + before_state = clean_env.wait_for_dev_task_terminal_state(task_id, "failed", timeout=60) + assert before_state["artwork_id"] == artwork_id + assert _attempt_history(before_state) == [1, 2] + + clean_env.docker_exec( + "pixivutil-worker", + "sh", + "-lc", + "touch /tmp/pixivutil-dev-dlq-success.flag", + ) + + resumed = clean_env.api_json("POST", f"/api/queue/dead-letter/{task_id}/resume") + assert resumed["requeued"] is True + assert resumed["task_name"] == "dev_download_artworks_by_id" + + after_state = clean_env.wait_for_dev_task_terminal_state(task_id, "succeeded", timeout=90) + assert _attempt_history(after_state) == [1, 2, 1, 2] + clean_env.wait_for_queue_count("pixivutil-queue", 0, timeout=90) + clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=90) + + @pytest.mark.pixiv_api def test_dlq_resume_all_replays_multiple_native_celery_messages(clean_env): + """ + Bulk DLQ replay should requeue all recognized native Celery messages and empty the dead letter queue. + + This covers the "resume all" operational path used to recover multiple failed tasks at once. If this fails, + operators may believe tasks were recovered while some messages remain stranded in DLQ or are not reprocessed. + """ task_a = clean_env.api_json("POST", "/api/dev/artwork/111111") task_b = clean_env.api_json("POST", "/api/dev/artwork/222222") @@ -53,7 +108,44 @@ def test_dlq_resume_all_replays_multiple_native_celery_messages(clean_env): resumed = clean_env.api_json("POST", "/api/queue/dead-letter/resume") assert resumed["requeued"] == 2 - clean_env.wait_worker_log_contains("Sentinel found at /tmp/pixivutil-dev-dlq-success.flag; succeeding on resumed run") + clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "succeeded", timeout=90) + clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "succeeded", timeout=90) clean_env.wait_for_queue_count("pixivutil-queue", 0, timeout=90) clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=90) assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] + + +@pytest.mark.pixiv_api +def test_dlq_resume_all_resets_retry_counter_for_replayed_native_celery_messages(clean_env): + """ + Bulk replay should reset retry state for every replayed native Celery message, not only the single-item path. + + The single-message and bulk replay endpoints should have consistent replay semantics. If this fails, bulk recovery + can silently requeue tasks with stale retry metadata, causing immediate re-failure and making DLQ recovery brittle. + """ + artwork_ids = (444444, 555555) + task_a = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_ids[0]}") + task_b = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_ids[1]}") + + clean_env.wait_for_queue_count("pixivutil-dead-letter", 2, timeout=90) + before_state_a = clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "failed", timeout=90) + before_state_b = clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "failed", timeout=90) + assert _attempt_history(before_state_a) == [1, 2] + assert _attempt_history(before_state_b) == [1, 2] + + clean_env.docker_exec( + "pixivutil-worker", + "sh", + "-lc", + "touch /tmp/pixivutil-dev-dlq-success.flag", + ) + + resumed = clean_env.api_json("POST", "/api/queue/dead-letter/resume") + assert resumed["requeued"] == 2 + + after_state_a = clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "succeeded", timeout=90) + after_state_b = clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "succeeded", timeout=90) + assert _attempt_history(after_state_a) == [1, 2, 1, 2] + assert _attempt_history(after_state_b) == [1, 2, 1, 2] + clean_env.wait_for_queue_count("pixivutil-queue", 0, timeout=90) + clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=90) diff --git a/tests/conftest.py b/tests/conftest.py index 268024a..a601177 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,31 +4,6 @@ import pytest -def pytest_addoption(parser: pytest.Parser): - parser.addoption( - "--pixiv-api", - action="store_true", - default=False, - help="Run tests that make Pixiv API calls.", - ) - - -def pytest_configure(config: pytest.Config): - config.addinivalue_line( - "markers", - "pixiv_api: test performs Pixiv API calls and is skipped unless --pixiv-api is provided.", - ) - - -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]): - if config.getoption("--pixiv-api"): - return - skip_pixiv_api = pytest.mark.skip(reason="need --pixiv-api option enabled") - for item in items: - if "pixiv_api" in item.keywords: - item.add_marker(skip_pixiv_api) - - @pytest.fixture def temp_dir(): """ From f109dc4835ca94edb6c550fa261133937c4026f6 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 15:56:52 -0800 Subject: [PATCH 14/32] implement priority queue --- PixivServer/config/celery.py | 9 ++- PixivServer/routers/dev.py | 44 ++++++++++- PixivServer/worker/dev.py | 95 ++++++++++++++++++++++++ integration_tests/conftest.py | 28 ++++++- integration_tests/test_queue_priority.py | 77 +++++++++++++++++++ tests/test_celery_config.py | 8 ++ 6 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 integration_tests/test_queue_priority.py diff --git a/PixivServer/config/celery.py b/PixivServer/config/celery.py index 36bcafa..517156f 100644 --- a/PixivServer/config/celery.py +++ b/PixivServer/config/celery.py @@ -17,7 +17,10 @@ exchange=default_exchange, routing_key='pixivutil-queue', durable=True, - queue_arguments={'x-dead-letter-exchange': 'pixivutil-dlx'}, + queue_arguments={ + 'x-dead-letter-exchange': 'pixivutil-dlx', + 'x-max-priority': 3, + }, ), ) @@ -26,3 +29,7 @@ CELERY_TASK_ACKS_LATE = True CELERY_ACKS_ON_FAILURE_OR_TIMEOUT = False CELERY_TASK_REJECT_ON_WORKER_LOST = True + +# Keep broker-side queue priority behavior observable with a single worker. +# Without this, Celery can reserve multiple low-priority tasks before later high-priority tasks arrive. +CELERYD_PREFETCH_MULTIPLIER = 1 diff --git a/PixivServer/routers/dev.py b/PixivServer/routers/dev.py index c9894e2..f705cef 100644 --- a/PixivServer/routers/dev.py +++ b/PixivServer/routers/dev.py @@ -1,11 +1,16 @@ import logging from celery.result import AsyncResult -from fastapi import APIRouter, HTTPException, Response +from fastapi import APIRouter, HTTPException, Query, Response from fastapi.responses import JSONResponse from PixivServer.models.pixiv_worker import DownloadArtworkByIdRequest -from PixivServer.worker.dev import dev_download_artworks_by_id, get_dev_task_state +from PixivServer.worker.dev import ( + dev_download_artworks_by_id, + dev_priority_probe_task, + get_dev_task_state, + get_priority_probe_state, +) logger = logging.getLogger('uvicorn.pixivutil') router = APIRouter() @@ -37,3 +42,38 @@ async def dev_task_status(task_id: str) -> Response: if state is None: raise HTTPException(status_code=404, detail=f"Dev task state not found: {task_id}") return JSONResponse(state) + + +@router.post("/priority/{label}") +async def dev_queue_priority_probe_task( + label: str, + priority: int = Query(default=2, ge=1, le=3), + sleep_ms: int = Query(default=1000, ge=0, le=10000), +) -> Response: + """ + (test) Enqueue a no-op task with explicit broker priority and predictable runtime. + """ + task: AsyncResult = dev_priority_probe_task.apply_async( + kwargs={ + "request_dict": { + "label": label, + "priority": priority, + "sleep_ms": sleep_ms, + }, + }, + priority=priority, + ) + return JSONResponse({ + "task_id": task.id, + "label": label, + "priority": priority, + "sleep_ms": sleep_ms, + }) + + +@router.get("/priority") +async def dev_priority_probe_status() -> Response: + """ + (test) Return execution ordering state for dev priority probe tasks. + """ + return JSONResponse(get_priority_probe_state()) diff --git a/PixivServer/worker/dev.py b/PixivServer/worker/dev.py index e6da03e..161a458 100644 --- a/PixivServer/worker/dev.py +++ b/PixivServer/worker/dev.py @@ -1,5 +1,6 @@ import json import logging +import time from pathlib import Path from typing import Any @@ -13,6 +14,7 @@ _SIMULATED_MAX_RETRIES = 1 _SUCCESS_SENTINEL_PATH = Path("/tmp/pixivutil-dev-dlq-success.flag") _TASK_STATE_PATH = Path("/workdir/.pixivUtil2/dev-dlq-task-state.json") +_PRIORITY_STATE_PATH = Path("/workdir/.pixivUtil2/dev-priority-task-state.json") def _load_task_states() -> dict[str, dict[str, Any]]: @@ -77,6 +79,84 @@ def get_dev_task_state(task_id: str) -> dict[str, Any] | None: state = _load_task_states().get(task_id) return state if isinstance(state, dict) else None + +def _load_priority_probe_state() -> dict[str, Any]: + try: + data = json.loads(_PRIORITY_STATE_PATH.read_text()) + except FileNotFoundError: + return {"started": [], "completed": [], "tasks": {}} + except json.JSONDecodeError: + logger.warning("(dev) Could not parse priority probe state file; resetting") + return {"started": [], "completed": [], "tasks": {}} + + if not isinstance(data, dict): + return {"started": [], "completed": [], "tasks": {}} + + started = data.get("started") + completed = data.get("completed") + tasks = data.get("tasks") + return { + "started": started if isinstance(started, list) else [], + "completed": completed if isinstance(completed, list) else [], + "tasks": tasks if isinstance(tasks, dict) else {}, + } + + +def _save_priority_probe_state(state: dict[str, Any]) -> None: + _PRIORITY_STATE_PATH.parent.mkdir(parents=True, exist_ok=True) + temp_path = _PRIORITY_STATE_PATH.with_suffix(".tmp") + temp_path.write_text(json.dumps(state, sort_keys=True)) + temp_path.replace(_PRIORITY_STATE_PATH) + + +def _record_priority_probe_started(task_id: str, label: str, priority: int) -> None: + state = _load_priority_probe_state() + started = state.get("started") + if not isinstance(started, list): + started = [] + started.append(label) + state["started"] = started + + tasks = state.get("tasks") + if not isinstance(tasks, dict): + tasks = {} + tasks[task_id] = { + "task_id": task_id, + "label": label, + "priority": priority, + "started_index": len(started) - 1, + "status": "started", + } + state["tasks"] = tasks + _save_priority_probe_state(state) + + +def _record_priority_probe_completed(task_id: str) -> None: + state = _load_priority_probe_state() + completed = state.get("completed") + if not isinstance(completed, list): + completed = [] + + tasks = state.get("tasks") + if not isinstance(tasks, dict): + tasks = {} + task_state = tasks.get(task_id) + if isinstance(task_state, dict): + label = task_state.get("label") + if isinstance(label, str): + completed.append(label) + task_state["status"] = "completed" + task_state["completed_index"] = len(completed) - 1 + tasks[task_id] = task_state + + state["completed"] = completed + state["tasks"] = tasks + _save_priority_probe_state(state) + + +def get_priority_probe_state() -> dict[str, Any]: + return _load_priority_probe_state() + # This endpoint exists to confirm DLQ functionality. # Use this for any kind of DLQ-related task and make any necessary changes in logic to prove/confirm hypotheses. # This will be commented out once DLQ is stable, so in the meantime do whatever you want with this endpoint, @@ -101,3 +181,18 @@ def dev_download_artworks_by_id(self, request_dict: dict): _record_terminal_state(task_id, "failed") logger.error(f"(dev) Max retries exceeded for artwork_id={request.artwork_id}, raising terminal failure for broker DLQ") raise ConnectionError("Simulated terminal failure after retries") + + +@shared_task(bind=True, name="dev_priority_probe_task", queue='pixivutil-queue') +def dev_priority_probe_task(self, request_dict: dict): + task_id = str(self.request.id) + label = str(request_dict.get("label", task_id)) + priority = int(request_dict.get("priority", 2)) + sleep_ms = int(request_dict.get("sleep_ms", 1000)) + + logger.error(f"(dev-priority) starting label={label} priority={priority} task_id={task_id}") + _record_priority_probe_started(task_id, label, priority) + time.sleep(max(sleep_ms, 0) / 1000) + _record_priority_probe_completed(task_id) + logger.error(f"(dev-priority) completed label={label} priority={priority} task_id={task_id}") + return True diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 19600d7..7c0a711 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -141,12 +141,38 @@ def wait_for_dev_task_terminal_state(self, task_id: str, terminal_state: str, ti f"(last_state={last_state}, last_error={last_error})" ) + def dev_priority_probe_state(self) -> dict: + state = self.api_json("GET", "/api/dev/priority") + if not isinstance(state, dict): + raise RuntimeError(f"Unexpected dev priority probe payload: {state!r}") + return state + + def wait_for_priority_probe_started_count(self, expected: int, timeout: int = 90) -> dict: + deadline = time.time() + timeout + last_state: dict | None = None + while time.time() < deadline: + state = self.dev_priority_probe_state() + last_state = state + started = state.get("started") + if isinstance(started, list) and len(started) >= expected: + return state + time.sleep(0.5) + raise RuntimeError( + f"Priority probe did not reach started_count>={expected} within {timeout}s (last_state={last_state})" + ) + def clear_state(self) -> None: self.docker_exec( "pixivutil-worker", "sh", "-lc", - f"rm -f {DEV_SUCCESS_SENTINEL} /workdir/.pixivUtil2/dev-dlq-task-state.json /workdir/.pixivUtil2/dev-dlq-task-state.tmp", + ( + f"rm -f {DEV_SUCCESS_SENTINEL} " + "/workdir/.pixivUtil2/dev-dlq-task-state.json " + "/workdir/.pixivUtil2/dev-dlq-task-state.tmp " + "/workdir/.pixivUtil2/dev-priority-task-state.json " + "/workdir/.pixivUtil2/dev-priority-task-state.tmp" + ), check=False, ) self.docker_exec( diff --git a/integration_tests/test_queue_priority.py b/integration_tests/test_queue_priority.py new file mode 100644 index 0000000..6c90184 --- /dev/null +++ b/integration_tests/test_queue_priority.py @@ -0,0 +1,77 @@ +import random + +import pytest + + +@pytest.mark.pixiv_api +def test_high_priority_tasks_are_consecutive_and_preceded_by_at_most_one_low(clean_env): + """ + A later-published pair of high-priority tasks should jump ahead of queued low-priority tasks. + + In live-worker mode (concurrency=1), one low-priority task may already be running or reserved. We accept that, + but the next queued work should prioritize highs. This test proves the operational guarantee that high-priority + tasks are delayed by at most one low task and are not interleaved with queued lows. + """ + sleep_ms = 1200 + low_labels = [f"L{i}" for i in range(1, 6)] + high_labels = ["H1", "H2"] + + for label in low_labels: + clean_env.api_json("POST", f"/api/dev/priority/{label}?priority=1&sleep_ms={sleep_ms}") + for label in high_labels: + clean_env.api_json("POST", f"/api/dev/priority/{label}?priority=3&sleep_ms={sleep_ms}") + + state = clean_env.wait_for_priority_probe_started_count(len(low_labels) + len(high_labels), timeout=120) + started = state.get("started") + assert isinstance(started, list) + + h1_index = started.index("H1") + h2_index = started.index("H2") + assert h2_index == h1_index + 1 + assert h1_index <= 1, f"Expected at most one low before highs, got order={started}" + + +@pytest.mark.pixiv_api +def test_three_tier_priority_orders_h_then_n_then_l_with_at_most_one_leading_low(clean_env): + """ + In live-worker mode, at most one low task may execute before queued priorities take effect. + + After that, queued work should drain by priority order: high -> normal -> low. We randomize submission order + (deterministically) to avoid accidentally testing only a sorted enqueue path. + """ + sleep_ms = 1000 + rng = random.Random(20260224) + + labels = [f"L{i}" for i in range(1, 11)] + labels += [f"N{i}" for i in range(1, 6)] + labels += [f"H{i}" for i in range(1, 6)] + rng.shuffle(labels) + + # Preserve the live-worker edge case we want to allow: one low can start before later priorities arrive. + first_low_index = next(i for i, label in enumerate(labels) if label.startswith("L")) + labels[0], labels[first_low_index] = labels[first_low_index], labels[0] + + for label in labels: + priority = 1 if label.startswith("L") else 2 if label.startswith("N") else 3 + clean_env.api_json("POST", f"/api/dev/priority/{label}?priority={priority}&sleep_ms={sleep_ms}") + + state = clean_env.wait_for_priority_probe_started_count(len(labels), timeout=180) + started = state.get("started") + assert isinstance(started, list) + assert len(started) == len(labels) + + leading_low_count = 1 if started and started[0].startswith("L") else 0 + body = started[leading_low_count:] + + h_block = [label for label in body if label.startswith("H")] + n_block = [label for label in body if label.startswith("N")] + l_block = [label for label in body if label.startswith("L")] + + expected = h_block + n_block + l_block + assert body == expected, f"Expected H* then N* then L* after optional leading L, got order={started}" + + assert len(h_block) == 5, f"Expected 5 high tasks, got {len(h_block)} in order={started}" + assert len(n_block) == 5, f"Expected 5 normal tasks, got {len(n_block)} in order={started}" + assert len(l_block) == 10 - leading_low_count, ( + f"Expected remaining low tasks to be {10 - leading_low_count}, got {len(l_block)} in order={started}" + ) diff --git a/tests/test_celery_config.py b/tests/test_celery_config.py index d739db0..db9ccec 100644 --- a/tests/test_celery_config.py +++ b/tests/test_celery_config.py @@ -17,4 +17,12 @@ def test_main_queue_declares_dead_letter_exchange(): assert "pixivutil-queue" in queues assert queues["pixivutil-queue"].queue_arguments == { "x-dead-letter-exchange": "pixivutil-dlx", + "x-max-priority": 3, } + + +def test_worker_prefetch_multiplier_is_one_for_priority_fairness(): + app = Celery("pixivutil-test") + app.config_from_object("PixivServer.config.celery") + + assert app.conf.worker_prefetch_multiplier == 1 From 87b1cd9454b721dce710eac6c4fade5fa964d648 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 16:38:12 -0800 Subject: [PATCH 15/32] implement priority and dlq v1 upgrade on worker init --- PixivServer/config/celery.py | 40 +++++++++++------- PixivServer/routers/dev.py | 3 +- PixivServer/routers/dlq.py | 4 +- PixivServer/service/metrics.py | 3 +- PixivServer/worker/__init__.py | 25 +++++++++++- PixivServer/worker/dev.py | 5 ++- PixivServer/worker/download.py | 9 ++-- PixivServer/worker/metadata.py | 9 ++-- integration_tests/conftest.py | 52 ++++++++++++++++++++++-- integration_tests/test_dlq_replay.py | 24 ++++++----- integration_tests/test_queue_priority.py | 50 +++++++++++++++++++++++ tests/test_celery_config.py | 10 +++-- 12 files changed, 185 insertions(+), 49 deletions(-) diff --git a/PixivServer/config/celery.py b/PixivServer/config/celery.py index 517156f..13f10ef 100644 --- a/PixivServer/config/celery.py +++ b/PixivServer/config/celery.py @@ -2,27 +2,37 @@ from PixivServer.config import rabbitmq -default_exchange = Exchange('pixivutil-exchange', type='direct', durable=True, delivery_mode=2) -dlx_exchange = Exchange('pixivutil-dlx', type='fanout', durable=True, delivery_mode=2) +LEGACY_MAIN_EXCHANGE_NAME = "pixivutil-exchange" +LEGACY_DLX_EXCHANGE_NAME = "pixivutil-dlx" +LEGACY_MAIN_QUEUE_NAME = "pixivutil-queue" + +MAIN_EXCHANGE_NAME = "pixivutil-v1-exchange" +DLX_EXCHANGE_NAME = "pixivutil-v1-dlx" +MAIN_QUEUE_NAME = "pixivutil-v1-queue" +MAIN_ROUTING_KEY = MAIN_QUEUE_NAME +DEAD_LETTER_QUEUE_NAME = "pixivutil-v1-dead-letter" +QUEUE_MAX_PRIORITY = 3 + +default_exchange = Exchange(MAIN_EXCHANGE_NAME, type='direct', durable=True, delivery_mode=2) +dlx_exchange = Exchange(DLX_EXCHANGE_NAME, type='fanout', durable=True, delivery_mode=2) +main_queue = Queue( + name=MAIN_QUEUE_NAME, + exchange=default_exchange, + routing_key=MAIN_ROUTING_KEY, + durable=True, + queue_arguments={ + 'x-dead-letter-exchange': DLX_EXCHANGE_NAME, + 'x-max-priority': QUEUE_MAX_PRIORITY, + }, +) dead_letter_queue = Queue( - name="pixivutil-dead-letter", + name=DEAD_LETTER_QUEUE_NAME, exchange=dlx_exchange, routing_key='', durable=True, ) -CELERY_QUEUES = ( - Queue( - name="pixivutil-queue", - exchange=default_exchange, - routing_key='pixivutil-queue', - durable=True, - queue_arguments={ - 'x-dead-letter-exchange': 'pixivutil-dlx', - 'x-max-priority': 3, - }, - ), -) +CELERY_QUEUES = (main_queue,) BROKER_URL = rabbitmq.config.broker_url CELERY_ACKS_LATE = True diff --git a/PixivServer/routers/dev.py b/PixivServer/routers/dev.py index f705cef..00e21ab 100644 --- a/PixivServer/routers/dev.py +++ b/PixivServer/routers/dev.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, HTTPException, Query, Response from fastapi.responses import JSONResponse +from PixivServer.config.celery import QUEUE_MAX_PRIORITY from PixivServer.models.pixiv_worker import DownloadArtworkByIdRequest from PixivServer.worker.dev import ( dev_download_artworks_by_id, @@ -47,7 +48,7 @@ async def dev_task_status(task_id: str) -> Response: @router.post("/priority/{label}") async def dev_queue_priority_probe_task( label: str, - priority: int = Query(default=2, ge=1, le=3), + priority: int = Query(default=2, ge=1, le=QUEUE_MAX_PRIORITY + 1), # +1: allow one example of an over-limit value for clamping tests sleep_ms: int = Query(default=1000, ge=0, le=10000), ) -> Response: """ diff --git a/PixivServer/routers/dlq.py b/PixivServer/routers/dlq.py index f36c4f0..19f7743 100644 --- a/PixivServer/routers/dlq.py +++ b/PixivServer/routers/dlq.py @@ -7,7 +7,7 @@ from kombu import Connection from PixivServer.config import rabbitmq -from PixivServer.config.celery import dead_letter_queue, default_exchange +from PixivServer.config.celery import MAIN_ROUTING_KEY, dead_letter_queue, default_exchange from PixivServer.worker import pixiv_worker logger = logging.getLogger('uvicorn.pixivutil') @@ -139,7 +139,7 @@ def _republish_native_celery_message(conn, msg) -> str | None: producer.publish( raw_body, exchange=default_exchange, - routing_key="pixivutil-queue", + routing_key=MAIN_ROUTING_KEY, headers=_clean_republish_headers(headers), content_type=msg.content_type, content_encoding=msg.content_encoding, diff --git a/PixivServer/service/metrics.py b/PixivServer/service/metrics.py index 93883f8..4867e04 100644 --- a/PixivServer/service/metrics.py +++ b/PixivServer/service/metrics.py @@ -20,6 +20,7 @@ import PixivServer.service.pixiv from PixivServer.config.pixivutil import config as pixivutil_config from PixivServer.config.rabbitmq import config as rabbitmq_config +from PixivServer.config.celery import MAIN_QUEUE_NAME from PixivServer.metrics import ( DB_ARTWORKS, DB_MEMBERS, @@ -98,7 +99,7 @@ def _collect_queue_depth() -> None: raw_vhost = parsed.path.lstrip("/") vhost = raw_vhost if raw_vhost else "/" encoded_vhost = quote(vhost, safe="") - url = f"http://{host}:15672/api/queues/{encoded_vhost}/pixivutil-queue" # TODO: if rabbitmq management layer isn't set up, then we have a problem. + url = f"http://{host}:15672/api/queues/{encoded_vhost}/{MAIN_QUEUE_NAME}" # TODO: if rabbitmq management layer isn't set up, then we have a problem. credentials = base64.b64encode(f"{user}:{password}".encode()).decode() req = urllib.request.Request(url, headers={"Authorization": f"Basic {credentials}"}) try: diff --git a/PixivServer/worker/__init__.py b/PixivServer/worker/__init__.py index 94c1bf1..56b04da 100644 --- a/PixivServer/worker/__init__.py +++ b/PixivServer/worker/__init__.py @@ -2,11 +2,16 @@ from celery import Celery from celery.signals import setup_logging, worker_init, worker_shutdown +from kombu import Queue import PixivServer import PixivServer.service import PixivServer.service.pixiv -from PixivServer.config.celery import dead_letter_queue +from PixivServer.config.celery import ( + LEGACY_MAIN_QUEUE_NAME, + dead_letter_queue, + main_queue, +) from PixivServer.config.server import config as server_config logger = logging.getLogger(__name__) @@ -18,6 +23,8 @@ @worker_init.connect def on_worker_init(sender, **kwargs): with sender.app.connection() as conn: + _cleanup_legacy_queue(conn, LEGACY_MAIN_QUEUE_NAME) + main_queue.bind(conn).declare() dead_letter_queue.bind(conn).declare() PixivServer.service.pixiv.service.open() @@ -33,6 +40,22 @@ def config_loggers(*args, **kwargs): return +def _cleanup_legacy_queue(conn, queue_name: str) -> None: + queue = Queue(name=queue_name, durable=True).bind(conn) + try: + queue.purge() + logger.warning(f"Purged legacy queue during v1 broker cutover: {queue_name}") + except Exception as exc: # noqa: BLE001 + if "NOT_FOUND" not in str(exc): + logger.warning(f"Failed to purge legacy queue {queue_name}: {exc}") + try: + queue.delete(if_unused=False, if_empty=False) + logger.warning(f"Deleted legacy queue during v1 broker cutover: {queue_name}") + except Exception as exc: # noqa: BLE001 + if "NOT_FOUND" not in str(exc): + logger.warning(f"Failed to delete legacy queue {queue_name}: {exc}") + + # @celery.on_after_configure.connect # def setup_periodic_tasks(sender, **kwargs): # sender.add_periodic_task(worker_config.subscription_time_seconds, run_artist_subscription_job.s(), name='Artist subscription job') diff --git a/PixivServer/worker/dev.py b/PixivServer/worker/dev.py index 161a458..7cd7664 100644 --- a/PixivServer/worker/dev.py +++ b/PixivServer/worker/dev.py @@ -6,6 +6,7 @@ from celery import shared_task +from PixivServer.config.celery import MAIN_QUEUE_NAME from PixivServer.models.pixiv_worker import DownloadArtworkByIdRequest logger = logging.getLogger(__name__) @@ -161,7 +162,7 @@ def get_priority_probe_state() -> dict[str, Any]: # Use this for any kind of DLQ-related task and make any necessary changes in logic to prove/confirm hypotheses. # This will be commented out once DLQ is stable, so in the meantime do whatever you want with this endpoint, # just clean it up when done and don't commit things back in. -@shared_task(bind=True, name="dev_download_artworks_by_id", queue='pixivutil-queue') +@shared_task(bind=True, name="dev_download_artworks_by_id", queue=MAIN_QUEUE_NAME) def dev_download_artworks_by_id(self, request_dict: dict): request = DownloadArtworkByIdRequest(**request_dict) task_id = str(self.request.id) @@ -183,7 +184,7 @@ def dev_download_artworks_by_id(self, request_dict: dict): raise ConnectionError("Simulated terminal failure after retries") -@shared_task(bind=True, name="dev_priority_probe_task", queue='pixivutil-queue') +@shared_task(bind=True, name="dev_priority_probe_task", queue=MAIN_QUEUE_NAME) def dev_priority_probe_task(self, request_dict: dict): task_id = str(self.request.id) label = str(request_dict.get("label", task_id)) diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py index f4449d1..fed9542 100644 --- a/PixivServer/worker/download.py +++ b/PixivServer/worker/download.py @@ -6,6 +6,7 @@ from celery import shared_task import PixivServer.service.pixiv +from PixivServer.config.celery import MAIN_QUEUE_NAME from PixivServer.models.pixiv_worker import ( DeleteArtworkByIdRequest, DownloadArtworkByIdRequest, @@ -21,7 +22,7 @@ def __job_sleep(): return 0 -@shared_task(name="download_artworks_by_id", queue='pixivutil-queue') +@shared_task(name="download_artworks_by_id", queue=MAIN_QUEUE_NAME) def download_artworks_by_id(request_dict: dict): try: request = DownloadArtworkByIdRequest(**request_dict) @@ -36,7 +37,7 @@ def download_artworks_by_id(request_dict: dict): __job_sleep() -@shared_task(name="download_artworks_by_member_id", queue='pixivutil-queue') +@shared_task(name="download_artworks_by_member_id", queue=MAIN_QUEUE_NAME) def download_artworks_by_member_id(request_dict: dict): try: request = DownloadArtworksByMemberIdRequest(**request_dict) @@ -51,7 +52,7 @@ def download_artworks_by_member_id(request_dict: dict): __job_sleep() -@shared_task(name="download_artworks_by_tag", queue='pixivutil-queue') +@shared_task(name="download_artworks_by_tag", queue=MAIN_QUEUE_NAME) def download_artworks_by_tag(request_dict: dict): try: request = DownloadArtworksByTagsRequest(**request_dict) @@ -66,7 +67,7 @@ def download_artworks_by_tag(request_dict: dict): __job_sleep() -@shared_task(name="delete_artwork_by_id", queue='pixivutil-queue') +@shared_task(name="delete_artwork_by_id", queue=MAIN_QUEUE_NAME) def delete_artwork_by_id(request_dict: dict): try: request = DeleteArtworkByIdRequest(**request_dict) diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py index bf23298..d26c6d0 100644 --- a/PixivServer/worker/metadata.py +++ b/PixivServer/worker/metadata.py @@ -6,6 +6,7 @@ from celery import shared_task import PixivServer.service.pixiv +from PixivServer.config.celery import MAIN_QUEUE_NAME from PixivServer.models.pixiv_worker import ( DownloadArtworkMetadataByIdRequest, DownloadMemberMetadataByIdRequest, @@ -21,7 +22,7 @@ def __job_sleep(): return 0 -@shared_task(name="download_member_metadata_by_id", queue='pixivutil-queue') +@shared_task(name="download_member_metadata_by_id", queue=MAIN_QUEUE_NAME) def download_member_metadata_by_id(request_dict: dict): try: request = DownloadMemberMetadataByIdRequest(**request_dict) @@ -39,7 +40,7 @@ def download_member_metadata_by_id(request_dict: dict): __job_sleep() -@shared_task(name="download_artwork_metadata_by_id", queue='pixivutil-queue') +@shared_task(name="download_artwork_metadata_by_id", queue=MAIN_QUEUE_NAME) def download_artwork_metadata_by_id(request_dict: dict): try: request = DownloadArtworkMetadataByIdRequest(**request_dict) @@ -57,7 +58,7 @@ def download_artwork_metadata_by_id(request_dict: dict): __job_sleep() -@shared_task(name="download_series_metadata_by_id", queue='pixivutil-queue') +@shared_task(name="download_series_metadata_by_id", queue=MAIN_QUEUE_NAME) def download_series_metadata_by_id(request_dict: dict): try: request = DownloadSeriesMetadataByIdRequest(**request_dict) @@ -75,7 +76,7 @@ def download_series_metadata_by_id(request_dict: dict): __job_sleep() -@shared_task(name="download_tag_metadata_by_id", queue='pixivutil-queue') +@shared_task(name="download_tag_metadata_by_id", queue=MAIN_QUEUE_NAME) def download_tag_metadata_by_id(request_dict: dict): try: request = DownloadTagMetadataByIdRequest(**request_dict) diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 7c0a711..bd5368a 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -8,6 +8,12 @@ import pytest +from PixivServer.config.celery import ( + DEAD_LETTER_QUEUE_NAME, + LEGACY_MAIN_QUEUE_NAME, + MAIN_QUEUE_NAME, +) + REPO_ROOT = Path(__file__).resolve().parents[1] BASE_URL = "http://localhost:8000" AUTH_HEADER = ("Authorization", "Bearer pixiv") @@ -100,6 +106,36 @@ def rabbitmq_queue_counts(self) -> dict[str, int]: continue return counts + def restart_worker(self) -> None: + self.compose("restart", "pixivutil-worker", timeout=240) + + def wait_for_queue_absent(self, queue_name: str, timeout: int = 45) -> None: + deadline = time.time() + timeout + last_counts: dict[str, int] | None = None + while time.time() < deadline: + counts = self.rabbitmq_queue_counts() + last_counts = counts + if queue_name not in counts: + return + time.sleep(1) + raise RuntimeError(f"Queue {queue_name!r} still exists after {timeout}s (last_counts={last_counts})") + + def seed_legacy_main_queue_message(self, body: str = "legacy-cutover-test") -> None: + script = ( + "from kombu import Connection, Exchange, Queue; " + f"from PixivServer.config.celery import LEGACY_MAIN_EXCHANGE_NAME, LEGACY_MAIN_QUEUE_NAME; " + "conn = Connection('amqp://guest:guest@rabbitmq:5672'); " + "conn.connect(); " + "ex = Exchange(LEGACY_MAIN_EXCHANGE_NAME, type='direct', durable=True); " + "q = Queue(LEGACY_MAIN_QUEUE_NAME, exchange=ex, routing_key=LEGACY_MAIN_QUEUE_NAME, durable=True); " + "q = q.bind(conn); q.declare(); " + "producer = conn.Producer(); " + f"producer.publish({body!r}, exchange=ex, routing_key=LEGACY_MAIN_QUEUE_NAME, " + "content_type='text/plain', content_encoding='utf-8', delivery_mode=2); " + "conn.release()" + ) + self.docker_exec("pixivutil-worker", "python", "-c", script, timeout=120) + def wait_for_queue_count(self, queue_name: str, expected: int, timeout: int = 45) -> None: deadline = time.time() + timeout last_count: int | None = None @@ -179,7 +215,15 @@ def clear_state(self) -> None: "rabbitmq", "rabbitmqctl", "purge_queue", - "pixivutil-queue", + MAIN_QUEUE_NAME, + check=False, + timeout=90, + ) + self.docker_exec( + "rabbitmq", + "rabbitmqctl", + "purge_queue", + DEAD_LETTER_QUEUE_NAME, check=False, timeout=90, ) @@ -187,7 +231,7 @@ def clear_state(self) -> None: "rabbitmq", "rabbitmqctl", "purge_queue", - "pixivutil-dead-letter", + LEGACY_MAIN_QUEUE_NAME, check=False, timeout=90, ) @@ -207,7 +251,7 @@ def compose_env() -> ComposeTestEnv: @pytest.fixture def clean_env(compose_env: ComposeTestEnv) -> ComposeTestEnv: compose_env.clear_state() - compose_env.wait_for_queue_count("pixivutil-queue", 0, timeout=30) - compose_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=30) + compose_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=30) + compose_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=30) yield compose_env compose_env.clear_state() diff --git a/integration_tests/test_dlq_replay.py b/integration_tests/test_dlq_replay.py index 327f002..5def066 100644 --- a/integration_tests/test_dlq_replay.py +++ b/integration_tests/test_dlq_replay.py @@ -1,5 +1,7 @@ import pytest +from PixivServer.config.celery import DEAD_LETTER_QUEUE_NAME, MAIN_QUEUE_NAME + def _attempt_history(state: dict) -> list[int]: history = state.get("attempt_history") @@ -19,7 +21,7 @@ def test_dlq_resume_replays_native_celery_message(clean_env): task = clean_env.api_json("POST", "/api/dev/artwork/424242") task_id = task["task_id"] - clean_env.wait_for_queue_count("pixivutil-dead-letter", 1, timeout=60) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 1, timeout=60) messages = clean_env.api_json("GET", "/api/queue/dead-letter/") assert len(messages) == 1 message = messages[0] @@ -40,7 +42,7 @@ def test_dlq_resume_replays_native_celery_message(clean_env): state = clean_env.wait_for_dev_task_terminal_state(task_id, "succeeded", timeout=90) assert _attempt_history(state) == [1, 2, 1, 2] - clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=60) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=60) assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] @@ -57,7 +59,7 @@ def test_dlq_resume_resets_retry_counter_for_replayed_native_celery_message(clea task = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_id}") task_id = task["task_id"] - clean_env.wait_for_queue_count("pixivutil-dead-letter", 1, timeout=60) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 1, timeout=60) before_state = clean_env.wait_for_dev_task_terminal_state(task_id, "failed", timeout=60) assert before_state["artwork_id"] == artwork_id assert _attempt_history(before_state) == [1, 2] @@ -75,8 +77,8 @@ def test_dlq_resume_resets_retry_counter_for_replayed_native_celery_message(clea after_state = clean_env.wait_for_dev_task_terminal_state(task_id, "succeeded", timeout=90) assert _attempt_history(after_state) == [1, 2, 1, 2] - clean_env.wait_for_queue_count("pixivutil-queue", 0, timeout=90) - clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=90) + clean_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=90) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=90) @pytest.mark.pixiv_api @@ -90,7 +92,7 @@ def test_dlq_resume_all_replays_multiple_native_celery_messages(clean_env): task_a = clean_env.api_json("POST", "/api/dev/artwork/111111") task_b = clean_env.api_json("POST", "/api/dev/artwork/222222") - clean_env.wait_for_queue_count("pixivutil-dead-letter", 2, timeout=90) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 2, timeout=90) messages = clean_env.api_json("GET", "/api/queue/dead-letter/") assert len(messages) == 2 @@ -110,8 +112,8 @@ def test_dlq_resume_all_replays_multiple_native_celery_messages(clean_env): clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "succeeded", timeout=90) clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "succeeded", timeout=90) - clean_env.wait_for_queue_count("pixivutil-queue", 0, timeout=90) - clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=90) + clean_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=90) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=90) assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] @@ -127,7 +129,7 @@ def test_dlq_resume_all_resets_retry_counter_for_replayed_native_celery_messages task_a = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_ids[0]}") task_b = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_ids[1]}") - clean_env.wait_for_queue_count("pixivutil-dead-letter", 2, timeout=90) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 2, timeout=90) before_state_a = clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "failed", timeout=90) before_state_b = clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "failed", timeout=90) assert _attempt_history(before_state_a) == [1, 2] @@ -147,5 +149,5 @@ def test_dlq_resume_all_resets_retry_counter_for_replayed_native_celery_messages after_state_b = clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "succeeded", timeout=90) assert _attempt_history(after_state_a) == [1, 2, 1, 2] assert _attempt_history(after_state_b) == [1, 2, 1, 2] - clean_env.wait_for_queue_count("pixivutil-queue", 0, timeout=90) - clean_env.wait_for_queue_count("pixivutil-dead-letter", 0, timeout=90) + clean_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=90) + clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=90) diff --git a/integration_tests/test_queue_priority.py b/integration_tests/test_queue_priority.py index 6c90184..52801a9 100644 --- a/integration_tests/test_queue_priority.py +++ b/integration_tests/test_queue_priority.py @@ -2,6 +2,8 @@ import pytest +from PixivServer.config.celery import LEGACY_MAIN_QUEUE_NAME, MAIN_QUEUE_NAME, QUEUE_MAX_PRIORITY + @pytest.mark.pixiv_api def test_high_priority_tasks_are_consecutive_and_preceded_by_at_most_one_low(clean_env): @@ -75,3 +77,51 @@ def test_three_tier_priority_orders_h_then_n_then_l_with_at_most_one_leading_low assert len(l_block) == 10 - leading_low_count, ( f"Expected remaining low tasks to be {10 - leading_low_count}, got {len(l_block)} in order={started}" ) + + +@pytest.mark.pixiv_api +def test_priority_values_above_queue_max_are_not_a_higher_tier_than_max(clean_env): + """ + RabbitMQ x-max-priority should clamp >max values so they behave like max priority, not a new tier. + """ + over_limit = QUEUE_MAX_PRIORITY + 1 + clean_env.api_json("POST", "/api/dev/priority/L1?priority=1&sleep_ms=1800") + clean_env.api_json("POST", "/api/dev/priority/H3?priority=3&sleep_ms=200") + clean_env.api_json(f"POST", f"/api/dev/priority/HOver?priority={over_limit}&sleep_ms=200") + clean_env.api_json("POST", "/api/dev/priority/N1?priority=2&sleep_ms=200") + + state = clean_env.wait_for_priority_probe_started_count(4, timeout=90) + started = state.get("started") + assert isinstance(started, list) + assert started[0] == "L1" + assert started[1:] == ["H3", "HOver", "N1"], f"Expected over-limit priority to be clamped to max-priority tier, got {started}" + + +@pytest.mark.pixiv_api +def test_worker_startup_deletes_legacy_main_queue(clean_env): + """ + Hard-cutover behavior: worker init should purge/delete the legacy main queue if it exists. + """ + clean_env.seed_legacy_main_queue_message() + clean_env.wait_for_queue_count(LEGACY_MAIN_QUEUE_NAME, 1, timeout=30) + + clean_env.restart_worker() + clean_env.wait_for_queue_absent(LEGACY_MAIN_QUEUE_NAME, timeout=60) + + +@pytest.mark.pixiv_api +def test_worker_restart_keeps_v1_queues_usable_when_they_already_exist(clean_env): + """ + Restarting the worker with pre-existing v1 queues should remain healthy and continue processing tasks. + """ + clean_env.api_json("POST", "/api/dev/priority/BeforeRestart?priority=2&sleep_ms=200") + clean_env.wait_for_priority_probe_started_count(1, timeout=60) + + clean_env.restart_worker() + + clean_env.api_json("POST", "/api/dev/priority/AfterRestart?priority=2&sleep_ms=200") + state = clean_env.wait_for_priority_probe_started_count(2, timeout=90) + started = state.get("started") + assert isinstance(started, list) + assert started[:2] == ["BeforeRestart", "AfterRestart"] + assert MAIN_QUEUE_NAME in clean_env.rabbitmq_queue_counts() diff --git a/tests/test_celery_config.py b/tests/test_celery_config.py index db9ccec..b3edee5 100644 --- a/tests/test_celery_config.py +++ b/tests/test_celery_config.py @@ -1,5 +1,7 @@ from celery import Celery +from PixivServer.config.celery import DLX_EXCHANGE_NAME, MAIN_QUEUE_NAME, QUEUE_MAX_PRIORITY + def test_celery_failure_is_rejected_for_rabbitmq_dlq(): app = Celery("pixivutil-test") @@ -14,10 +16,10 @@ def test_main_queue_declares_dead_letter_exchange(): app.config_from_object("PixivServer.config.celery") queues = {queue.name: queue for queue in app.conf.CELERY_QUEUES} - assert "pixivutil-queue" in queues - assert queues["pixivutil-queue"].queue_arguments == { - "x-dead-letter-exchange": "pixivutil-dlx", - "x-max-priority": 3, + assert MAIN_QUEUE_NAME in queues + assert queues[MAIN_QUEUE_NAME].queue_arguments == { + "x-dead-letter-exchange": DLX_EXCHANGE_NAME, + "x-max-priority": QUEUE_MAX_PRIORITY, } From 21752ea5a519654b4fd33de2a2f7b42122ad9fc6 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 17:03:41 -0800 Subject: [PATCH 16/32] implement priority queue function to all endpoints and clients --- PixivServer/routers/download_queue.py | 28 ++++++++--- PixivServer/routers/metadata_queue.py | 27 +++++++--- PixivUtilClient/pixivutil_client/client.py | 58 +++++++++++++++++----- 3 files changed, 84 insertions(+), 29 deletions(-) diff --git a/PixivServer/routers/download_queue.py b/PixivServer/routers/download_queue.py index 870b574..0ec6cd5 100644 --- a/PixivServer/routers/download_queue.py +++ b/PixivServer/routers/download_queue.py @@ -5,10 +5,11 @@ from datetime import timedelta from celery.result import AsyncResult -from fastapi import APIRouter, Response +from fastapi import APIRouter, Query, Response from fastapi.responses import JSONResponse from pixivutil_server_common.models import TagSortOrder, TagTypeMode +from PixivServer.config.celery import QUEUE_MAX_PRIORITY from PixivServer.models.pixiv_worker import ( DeleteArtworkByIdRequest, DownloadArtworkByIdRequest, @@ -58,14 +59,17 @@ def get_member_name_from_db(member_id: int) -> str | None: @router.post("/artwork/{artwork_id}") -async def queue_download_artwork_by_id(artwork_id: str) -> Response: +async def queue_download_artwork_by_id( + artwork_id: str, + priority: int = Query(default=QUEUE_MAX_PRIORITY, ge=1, le=QUEUE_MAX_PRIORITY), +) -> Response: """ Download Pixiv image by ID. """ logger.info(f"Downloading Pixiv artwork by image ID: {artwork_id}.") request = DownloadArtworkByIdRequest(artwork_id=int(artwork_id)) artwork_title, member_name = get_artwork_and_member_name_from_db(request.artwork_id) - task: AsyncResult = download_artworks_by_id.delay(request.model_dump()) + task: AsyncResult = download_artworks_by_id.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ "task_id": task.id, 'artwork_id': artwork_id, @@ -74,14 +78,17 @@ async def queue_download_artwork_by_id(artwork_id: str) -> Response: }) @router.post("/member/{member_id}") -async def queue_download_artworks_by_member_id(member_id: str) -> Response: +async def queue_download_artworks_by_member_id( + member_id: str, + priority: int = Query(default=2, ge=1, le=QUEUE_MAX_PRIORITY), +) -> Response: """ Download Pixiv image by member ID. """ logger.info(f"Downloading Pixiv artworks by member ID: {member_id}.") request = DownloadArtworksByMemberIdRequest(member_id=int(member_id)) member_name = get_member_name_from_db(request.member_id) - task: AsyncResult = download_artworks_by_member_id.delay(request.model_dump()) + task: AsyncResult = download_artworks_by_member_id.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ "task_id": task.id, 'member_id': member_id, @@ -98,6 +105,7 @@ async def queue_download_artworks_by_tag( start_date: str | None = None, end_date: str | None = None, lookback_days: int | None = None, + priority: int = Query(default=1, ge=1, le=QUEUE_MAX_PRIORITY), ) -> Response: """ Download Pixiv images that have a given tag. @@ -134,14 +142,18 @@ async def queue_download_artworks_by_tag( start_date=start_date, end_date=end_date, ) - task: AsyncResult = download_artworks_by_tag.delay(request.model_dump()) + task: AsyncResult = download_artworks_by_tag.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ 'task_id': task.id, 'tag': decoded_tag, }) @router.delete("/artwork/{artwork_id}") -async def queue_delete_artwork_by_id(artwork_id: str, delete_metadata: bool = True) -> Response: +async def queue_delete_artwork_by_id( + artwork_id: str, + delete_metadata: bool = True, + priority: int = Query(default=2, ge=1, le=QUEUE_MAX_PRIORITY), +) -> Response: """ Delete Pixiv image by ID from database and filesystem. @@ -154,7 +166,7 @@ async def queue_delete_artwork_by_id(artwork_id: str, delete_metadata: bool = Tr """ logger.info(f"Deleting Pixiv artwork by image ID: {artwork_id} (delete_metadata={delete_metadata}).") request = DeleteArtworkByIdRequest(artwork_id=int(artwork_id), delete_metadata=delete_metadata) - task: AsyncResult = delete_artwork_by_id.delay(request.model_dump()) + task: AsyncResult = delete_artwork_by_id.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ "task_id": task.id, 'artwork_id': artwork_id, diff --git a/PixivServer/routers/metadata_queue.py b/PixivServer/routers/metadata_queue.py index 1ac1fe7..04c026e 100644 --- a/PixivServer/routers/metadata_queue.py +++ b/PixivServer/routers/metadata_queue.py @@ -2,10 +2,11 @@ import urllib.parse from celery.result import AsyncResult -from fastapi import APIRouter +from fastapi import APIRouter, Query from fastapi.responses import JSONResponse from pixivutil_server_common.models import TagMetadataFilterMode +from PixivServer.config.celery import QUEUE_MAX_PRIORITY from PixivServer.models.pixiv_worker import ( DownloadArtworkMetadataByIdRequest, DownloadMemberMetadataByIdRequest, @@ -24,7 +25,10 @@ @router.post("/member/{member_id}") -async def queue_download_member_metadata_by_id(member_id: str) -> JSONResponse: +async def queue_download_member_metadata_by_id( + member_id: str, + priority: int = Query(default=2, ge=1, le=QUEUE_MAX_PRIORITY), +) -> JSONResponse: """ Queue download of member metadata by ID. """ @@ -36,12 +40,15 @@ async def queue_download_member_metadata_by_id(member_id: str) -> JSONResponse: member_id_int = int(member_id) logger.info(f"Queueing member metadata download by ID: {member_id_int}.") request = DownloadMemberMetadataByIdRequest(member_id=member_id_int) - task: AsyncResult = download_member_metadata_by_id.delay(request.model_dump()) + task: AsyncResult = download_member_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({"task_id": task.id, "member_id": member_id_int}) @router.post("/artwork/{artwork_id}") -async def queue_download_artwork_metadata_by_id(artwork_id: str) -> JSONResponse: +async def queue_download_artwork_metadata_by_id( + artwork_id: str, + priority: int = Query(default=2, ge=1, le=QUEUE_MAX_PRIORITY), +) -> JSONResponse: """ Queue download of artwork metadata by ID. """ @@ -53,12 +60,15 @@ async def queue_download_artwork_metadata_by_id(artwork_id: str) -> JSONResponse artwork_id_int = int(artwork_id) logger.info(f"Queueing artwork metadata download by ID: {artwork_id_int}.") request = DownloadArtworkMetadataByIdRequest(artwork_id=artwork_id_int) - task: AsyncResult = download_artwork_metadata_by_id.delay(request.model_dump()) + task: AsyncResult = download_artwork_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({"task_id": task.id, "artwork_id": artwork_id_int}) @router.post("/series/{series_id}") -async def queue_download_series_metadata_by_id(series_id: str) -> JSONResponse: +async def queue_download_series_metadata_by_id( + series_id: str, + priority: int = Query(default=1, ge=1, le=QUEUE_MAX_PRIORITY), +) -> JSONResponse: """ Queue download of series metadata by ID. """ @@ -70,7 +80,7 @@ async def queue_download_series_metadata_by_id(series_id: str) -> JSONResponse: series_id_int = int(series_id) logger.info(f"Queueing series metadata download by ID: {series_id_int}.") request = DownloadSeriesMetadataByIdRequest(series_id=series_id_int) - task: AsyncResult = download_series_metadata_by_id.delay(request.model_dump()) + task: AsyncResult = download_series_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({"task_id": task.id, "series_id": series_id_int}) @@ -78,6 +88,7 @@ async def queue_download_series_metadata_by_id(series_id: str) -> JSONResponse: async def queue_download_tag_metadata_by_id( tag: str, filter_mode: TagMetadataFilterMode = "none", + priority: int = Query(default=1, ge=1, le=QUEUE_MAX_PRIORITY), ) -> JSONResponse: """ Queue download of tag metadata by tag ID/name. @@ -89,7 +100,7 @@ async def queue_download_tag_metadata_by_id( request = DownloadTagMetadataByIdRequest( tag=decoded_tag, filter_mode=filter_mode ) - task: AsyncResult = download_tag_metadata_by_id.delay(request.model_dump()) + task: AsyncResult = download_tag_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse( {"task_id": task.id, "tag": decoded_tag, "filter_mode": request.filter_mode} ) diff --git a/PixivUtilClient/pixivutil_client/client.py b/PixivUtilClient/pixivutil_client/client.py index 9edef4e..382d681 100644 --- a/PixivUtilClient/pixivutil_client/client.py +++ b/PixivUtilClient/pixivutil_client/client.py @@ -128,12 +128,18 @@ async def health_pixiv(self) -> str: payload = await self._request("GET", "/api/health/pixiv") return str(payload) - async def queue_download_artwork(self, artwork_id: int) -> QueueTaskResponse: - payload = await self._request("POST", f"/api/queue/download/artwork/{artwork_id}") + async def queue_download_artwork(self, artwork_id: int, *, priority: int | None = None) -> QueueTaskResponse: + params: dict[str, Any] = {} + if priority is not None: + params["priority"] = priority + payload = await self._request("POST", f"/api/queue/download/artwork/{artwork_id}", params=params or None) return QueueTaskResponse.model_validate(payload) - async def queue_download_member(self, member_id: int) -> QueueTaskResponse: - payload = await self._request("POST", f"/api/queue/download/member/{member_id}") + async def queue_download_member(self, member_id: int, *, priority: int | None = None) -> QueueTaskResponse: + params: dict[str, Any] = {} + if priority is not None: + params["priority"] = priority + payload = await self._request("POST", f"/api/queue/download/member/{member_id}", params=params or None) return QueueTaskResponse.model_validate(payload) async def queue_download_tag( @@ -147,6 +153,7 @@ async def queue_download_tag( start_date: str | None = None, end_date: str | None = None, lookback_days: int | None = None, + priority: int | None = None, ) -> QueueTaskResponse: params: dict[str, Any] = { "sort_order": sort_order, @@ -161,41 +168,66 @@ async def queue_download_tag( params["end_date"] = end_date if lookback_days is not None: params["lookback_days"] = lookback_days + if priority is not None: + params["priority"] = priority encoded_tag = quote(tag, safe="") payload = await self._request("POST", f"/api/queue/download/tag/{encoded_tag}", params=params) return QueueTaskResponse.model_validate(payload) - async def queue_delete_artwork(self, artwork_id: int, delete_metadata: bool = True) -> QueueTaskResponse: + async def queue_delete_artwork( + self, + artwork_id: int, + delete_metadata: bool = True, + *, + priority: int | None = None, + ) -> QueueTaskResponse: + params: dict[str, Any] = {"delete_metadata": str(delete_metadata).lower()} + if priority is not None: + params["priority"] = priority payload = await self._request( "DELETE", f"/api/queue/download/artwork/{artwork_id}", - params={"delete_metadata": str(delete_metadata).lower()}, + params=params, ) return QueueTaskResponse.model_validate(payload) - async def queue_metadata_artwork(self, artwork_id: int) -> QueueTaskResponse: - payload = await self._request("POST", f"/api/queue/metadata/artwork/{artwork_id}") + async def queue_metadata_artwork(self, artwork_id: int, *, priority: int | None = None) -> QueueTaskResponse: + params: dict[str, Any] = {} + if priority is not None: + params["priority"] = priority + payload = await self._request("POST", f"/api/queue/metadata/artwork/{artwork_id}", params=params or None) return QueueTaskResponse.model_validate(payload) - async def queue_metadata_member(self, member_id: int) -> QueueTaskResponse: - payload = await self._request("POST", f"/api/queue/metadata/member/{member_id}") + async def queue_metadata_member(self, member_id: int, *, priority: int | None = None) -> QueueTaskResponse: + params: dict[str, Any] = {} + if priority is not None: + params["priority"] = priority + payload = await self._request("POST", f"/api/queue/metadata/member/{member_id}", params=params or None) return QueueTaskResponse.model_validate(payload) - async def queue_metadata_series(self, series_id: int) -> QueueTaskResponse: - payload = await self._request("POST", f"/api/queue/metadata/series/{series_id}") + async def queue_metadata_series(self, series_id: int, *, priority: int | None = None) -> QueueTaskResponse: + params: dict[str, Any] = {} + if priority is not None: + params["priority"] = priority + payload = await self._request("POST", f"/api/queue/metadata/series/{series_id}", params=params or None) return QueueTaskResponse.model_validate(payload) async def queue_metadata_tag( self, tag: str, filter_mode: TagMetadataFilterMode = "none", + *, + priority: int | None = None, ) -> QueueTaskResponse: + params: dict[str, Any] = {"filter_mode": filter_mode} + if priority is not None: + params["priority"] = priority encoded_tag = quote(tag, safe="") payload = await self._request( "POST", f"/api/queue/metadata/tag/{encoded_tag}", - params={"filter_mode": filter_mode}, + params=params, ) return QueueTaskResponse.model_validate(payload) From 711527df678d8fa473d1ff0e3dd5dffe268f2404 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 17:29:35 -0800 Subject: [PATCH 17/32] implement dead letter queue logic for prod task endpoints --- PixivServer/worker/download.py | 43 ++++++++++++++++++++++++---------- PixivServer/worker/metadata.py | 43 ++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py index fed9542..a80e17a 100644 --- a/PixivServer/worker/download.py +++ b/PixivServer/worker/download.py @@ -2,6 +2,7 @@ import random import time import traceback +from urllib.error import URLError from celery import shared_task @@ -13,17 +14,27 @@ DownloadArtworksByMemberIdRequest, DownloadArtworksByTagsRequest, ) +from PixivServer.service.pixiv import PixivException logger = logging.getLogger(__name__) +_NETWORK_MAX_RETRIES = 3 +_NETWORK_RETRY_COUNTDOWN = 60 + def __job_sleep(): time.sleep(random.uniform(1, 5)) return 0 -@shared_task(name="download_artworks_by_id", queue=MAIN_QUEUE_NAME) -def download_artworks_by_id(request_dict: dict): +def _is_network_exception(exc: BaseException) -> bool: + if isinstance(exc, PixivException): + return exc.errorCode in (PixivException.DOWNLOAD_FAILED_NETWORK, PixivException.SERVER_ERROR) + return isinstance(exc, (ConnectionError, TimeoutError, URLError)) + + +@shared_task(bind=True, name="download_artworks_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def download_artworks_by_id(self, request_dict: dict): try: request = DownloadArtworkByIdRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by ID: {request.artwork_id}.") @@ -32,13 +43,15 @@ def download_artworks_by_id(request_dict: dict): except Exception as e: logger.error(f"Error in download_artworks_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() -@shared_task(name="download_artworks_by_member_id", queue=MAIN_QUEUE_NAME) -def download_artworks_by_member_id(request_dict: dict): +@shared_task(bind=True, name="download_artworks_by_member_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def download_artworks_by_member_id(self, request_dict: dict): try: request = DownloadArtworksByMemberIdRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artworks by member ID: {request.member_id}.") @@ -47,13 +60,15 @@ def download_artworks_by_member_id(request_dict: dict): except Exception as e: logger.error(f"Error in download_artworks_by_member_id worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() -@shared_task(name="download_artworks_by_tag", queue=MAIN_QUEUE_NAME) -def download_artworks_by_tag(request_dict: dict): +@shared_task(bind=True, name="download_artworks_by_tag", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def download_artworks_by_tag(self, request_dict: dict): try: request = DownloadArtworksByTagsRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by tag: {request.tags}. Bookmark minimum: {request.bookmark_count}") @@ -62,13 +77,15 @@ def download_artworks_by_tag(request_dict: dict): except Exception as e: logger.error(f"Error in download_artworks_by_tag worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() -@shared_task(name="delete_artwork_by_id", queue=MAIN_QUEUE_NAME) -def delete_artwork_by_id(request_dict: dict): +@shared_task(bind=True, name="delete_artwork_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def delete_artwork_by_id(self, request_dict: dict): try: request = DeleteArtworkByIdRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Deleting artwork by ID: {request.artwork_id}.") @@ -77,6 +94,8 @@ def delete_artwork_by_id(request_dict: dict): except Exception as e: logger.error(f"Error in delete_artwork_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py index d26c6d0..7b7851a 100644 --- a/PixivServer/worker/metadata.py +++ b/PixivServer/worker/metadata.py @@ -2,6 +2,7 @@ import random import time import traceback +from urllib.error import URLError from celery import shared_task @@ -13,17 +14,27 @@ DownloadSeriesMetadataByIdRequest, DownloadTagMetadataByIdRequest, ) +from PixivServer.service.pixiv import PixivException logger = logging.getLogger(__name__) +_NETWORK_MAX_RETRIES = 3 +_NETWORK_RETRY_COUNTDOWN = 60 + def __job_sleep(): time.sleep(random.uniform(1, 5)) return 0 -@shared_task(name="download_member_metadata_by_id", queue=MAIN_QUEUE_NAME) -def download_member_metadata_by_id(request_dict: dict): +def _is_network_exception(exc: BaseException) -> bool: + if isinstance(exc, PixivException): + return exc.errorCode in (PixivException.DOWNLOAD_FAILED_NETWORK, PixivException.SERVER_ERROR) + return isinstance(exc, (ConnectionError, TimeoutError, URLError)) + + +@shared_task(bind=True, name="download_member_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def download_member_metadata_by_id(self, request_dict: dict): try: request = DownloadMemberMetadataByIdRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log( @@ -35,13 +46,15 @@ def download_member_metadata_by_id(request_dict: dict): except Exception as e: logger.error(f"Error in download_member_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() -@shared_task(name="download_artwork_metadata_by_id", queue=MAIN_QUEUE_NAME) -def download_artwork_metadata_by_id(request_dict: dict): +@shared_task(bind=True, name="download_artwork_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def download_artwork_metadata_by_id(self, request_dict: dict): try: request = DownloadArtworkMetadataByIdRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log( @@ -53,13 +66,15 @@ def download_artwork_metadata_by_id(request_dict: dict): except Exception as e: logger.error(f"Error in download_artwork_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() -@shared_task(name="download_series_metadata_by_id", queue=MAIN_QUEUE_NAME) -def download_series_metadata_by_id(request_dict: dict): +@shared_task(bind=True, name="download_series_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def download_series_metadata_by_id(self, request_dict: dict): try: request = DownloadSeriesMetadataByIdRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log( @@ -71,13 +86,15 @@ def download_series_metadata_by_id(request_dict: dict): except Exception as e: logger.error(f"Error in download_series_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() -@shared_task(name="download_tag_metadata_by_id", queue=MAIN_QUEUE_NAME) -def download_tag_metadata_by_id(request_dict: dict): +@shared_task(bind=True, name="download_tag_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +def download_tag_metadata_by_id(self, request_dict: dict): try: request = DownloadTagMetadataByIdRequest(**request_dict) PixivServer.service.pixiv.PixivHelper.print_and_log( @@ -89,6 +106,8 @@ def download_tag_metadata_by_id(request_dict: dict): except Exception as e: logger.error(f"Error in download_tag_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - raise + if _is_network_exception(e): + raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + return False finally: __job_sleep() From fa640d3f5e334017bcc2be98032cc71322f80e22 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 17:34:16 -0800 Subject: [PATCH 18/32] update metrics for DLQ --- PixivServer/metrics.py | 1 + PixivServer/service/metrics.py | 24 +++++++++++++++++++----- PixivServer/worker/download.py | 8 ++++---- PixivServer/worker/metadata.py | 8 ++++---- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/PixivServer/metrics.py b/PixivServer/metrics.py index 2d2aa6c..6943b6e 100644 --- a/PixivServer/metrics.py +++ b/PixivServer/metrics.py @@ -23,6 +23,7 @@ # --- Worker queue metrics (periodic) --- QUEUE_DEPTH = Gauge("pixivutil_queue_depth", "Number of messages pending in the task queue") +DLQ_DEPTH = Gauge("pixivutil_dlq_depth", "Number of messages in the dead letter queue") # --- Request metrics (per-request via middleware) --- HTTP_REQUESTS_TOTAL = Counter( diff --git a/PixivServer/service/metrics.py b/PixivServer/service/metrics.py index 4867e04..f2665a8 100644 --- a/PixivServer/service/metrics.py +++ b/PixivServer/service/metrics.py @@ -20,7 +20,7 @@ import PixivServer.service.pixiv from PixivServer.config.pixivutil import config as pixivutil_config from PixivServer.config.rabbitmq import config as rabbitmq_config -from PixivServer.config.celery import MAIN_QUEUE_NAME +from PixivServer.config.celery import DEAD_LETTER_QUEUE_NAME, MAIN_QUEUE_NAME from PixivServer.metrics import ( DB_ARTWORKS, DB_MEMBERS, @@ -29,6 +29,7 @@ DB_TAGS, DISK_DATABASE_BYTES, DISK_DOWNLOADS_BYTES, + DLQ_DEPTH, QUEUE_DEPTH, SYS_CPU_PERCENT, SYS_DISK_TOTAL_BYTES, @@ -91,7 +92,7 @@ def _collect_disk_metrics() -> None: DISK_DOWNLOADS_BYTES.set(total) -def _collect_queue_depth() -> None: +def _rabbitmq_queue_message_count(queue_name: str) -> int | None: parsed = urlparse(rabbitmq_config.broker_url) user = parsed.username or "guest" password = parsed.password or "guest" @@ -99,15 +100,27 @@ def _collect_queue_depth() -> None: raw_vhost = parsed.path.lstrip("/") vhost = raw_vhost if raw_vhost else "/" encoded_vhost = quote(vhost, safe="") - url = f"http://{host}:15672/api/queues/{encoded_vhost}/{MAIN_QUEUE_NAME}" # TODO: if rabbitmq management layer isn't set up, then we have a problem. + url = f"http://{host}:15672/api/queues/{encoded_vhost}/{queue_name}" # TODO: if rabbitmq management layer isn't set up, then we have a problem. credentials = base64.b64encode(f"{user}:{password}".encode()).decode() req = urllib.request.Request(url, headers={"Authorization": f"Basic {credentials}"}) try: with urllib.request.urlopen(req, timeout=5) as resp: data = json.loads(resp.read()) - QUEUE_DEPTH.set(data.get("messages", 0)) + return int(data.get("messages", 0)) except (urllib.error.URLError, OSError, ValueError): - pass # Management API unavailable; leave metric stale + return None # Management API unavailable + + +def _collect_queue_depth() -> None: + count = _rabbitmq_queue_message_count(MAIN_QUEUE_NAME) + if count is not None: + QUEUE_DEPTH.set(count) + + +def _collect_dlq_depth() -> None: + count = _rabbitmq_queue_message_count(DEAD_LETTER_QUEUE_NAME) + if count is not None: + DLQ_DEPTH.set(count) async def periodic_metrics_collector() -> None: @@ -129,6 +142,7 @@ async def periodic_metrics_collector() -> None: last_disk = time.monotonic() if now - last_queue >= _QUEUE_COLLECT_INTERVAL: await asyncio.to_thread(_collect_queue_depth) + await asyncio.to_thread(_collect_dlq_depth) last_queue = time.monotonic() except Exception: # noqa: BLE001 logger.warning(f"Metrics collector error: {traceback.format_exc()}") diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py index a80e17a..4fc8ff8 100644 --- a/PixivServer/worker/download.py +++ b/PixivServer/worker/download.py @@ -40,7 +40,7 @@ def download_artworks_by_id(self, request_dict: dict): PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by ID: {request.artwork_id}.") PixivServer.service.pixiv.service.download_artwork_by_id(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artworks_by_id worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): @@ -57,7 +57,7 @@ def download_artworks_by_member_id(self, request_dict: dict): PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artworks by member ID: {request.member_id}.") PixivServer.service.pixiv.service.download_artworks_by_member_id(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artworks_by_member_id worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): @@ -74,7 +74,7 @@ def download_artworks_by_tag(self, request_dict: dict): PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Downloading artwork by tag: {request.tags}. Bookmark minimum: {request.bookmark_count}") PixivServer.service.pixiv.service.download_artworks_by_tag(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artworks_by_tag worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): @@ -91,7 +91,7 @@ def delete_artwork_by_id(self, request_dict: dict): PixivServer.service.pixiv.PixivHelper.print_and_log("info", f"Deleting artwork by ID: {request.artwork_id}.") PixivServer.service.pixiv.service.delete_artwork_by_id(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in delete_artwork_by_id worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py index 7b7851a..3fa2bb8 100644 --- a/PixivServer/worker/metadata.py +++ b/PixivServer/worker/metadata.py @@ -43,7 +43,7 @@ def download_member_metadata_by_id(self, request_dict: dict): ) PixivServer.service.pixiv.service.download_member_metadata_by_id(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in download_member_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): @@ -63,7 +63,7 @@ def download_artwork_metadata_by_id(self, request_dict: dict): ) PixivServer.service.pixiv.service.download_artwork_metadata_by_id(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artwork_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): @@ -83,7 +83,7 @@ def download_series_metadata_by_id(self, request_dict: dict): ) PixivServer.service.pixiv.service.download_series_metadata_by_id(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in download_series_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): @@ -103,7 +103,7 @@ def download_tag_metadata_by_id(self, request_dict: dict): ) PixivServer.service.pixiv.service.download_tag_metadata_by_id(request) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error(f"Error in download_tag_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) if _is_network_exception(e): From 3502c31d959fcd0d2d95f8be82052c4afba16ecf Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 24 Feb 2026 18:14:58 -0800 Subject: [PATCH 19/32] clean up stuff --- PixivServer/config/celery.py | 1 - PixivServer/service/metrics.py | 15 +++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/PixivServer/config/celery.py b/PixivServer/config/celery.py index 13f10ef..87a0296 100644 --- a/PixivServer/config/celery.py +++ b/PixivServer/config/celery.py @@ -3,7 +3,6 @@ from PixivServer.config import rabbitmq LEGACY_MAIN_EXCHANGE_NAME = "pixivutil-exchange" -LEGACY_DLX_EXCHANGE_NAME = "pixivutil-dlx" LEGACY_MAIN_QUEUE_NAME = "pixivutil-queue" MAIN_EXCHANGE_NAME = "pixivutil-v1-exchange" diff --git a/PixivServer/service/metrics.py b/PixivServer/service/metrics.py index f2665a8..8c10a71 100644 --- a/PixivServer/service/metrics.py +++ b/PixivServer/service/metrics.py @@ -93,14 +93,17 @@ def _collect_disk_metrics() -> None: def _rabbitmq_queue_message_count(queue_name: str) -> int | None: - parsed = urlparse(rabbitmq_config.broker_url) - user = parsed.username or "guest" - password = parsed.password or "guest" - host = parsed.hostname or "rabbitmq" - raw_vhost = parsed.path.lstrip("/") + parsed_mgmt = urlparse(rabbitmq_config.management_url) + user = parsed_mgmt.username or "guest" + password = parsed_mgmt.password or "guest" + base = f"{parsed_mgmt.scheme}://{parsed_mgmt.hostname}" + if parsed_mgmt.port: + base = f"{base}:{parsed_mgmt.port}" + parsed_broker = urlparse(rabbitmq_config.broker_url) + raw_vhost = parsed_broker.path.lstrip("/") vhost = raw_vhost if raw_vhost else "/" encoded_vhost = quote(vhost, safe="") - url = f"http://{host}:15672/api/queues/{encoded_vhost}/{queue_name}" # TODO: if rabbitmq management layer isn't set up, then we have a problem. + url = f"{base}/api/queues/{encoded_vhost}/{queue_name}" credentials = base64.b64encode(f"{user}:{password}".encode()).decode() req = urllib.request.Request(url, headers={"Authorization": f"Basic {credentials}"}) try: From fcff8325c022309333908679d4ef0f38d4a7a021 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 25 Feb 2026 00:12:05 -0800 Subject: [PATCH 20/32] fix pyright typing for celery --- PixivServer/models/pixiv_worker.py | 11 +++++++++++ PixivServer/routers/download_queue.py | 17 +++++++++-------- PixivServer/routers/metadata_queue.py | 16 ++++++++-------- PixivServer/worker/download.py | 7 +++++++ PixivServer/worker/metadata.py | 7 +++++++ 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/PixivServer/models/pixiv_worker.py b/PixivServer/models/pixiv_worker.py index 4165827..825d06c 100644 --- a/PixivServer/models/pixiv_worker.py +++ b/PixivServer/models/pixiv_worker.py @@ -2,6 +2,9 @@ Model layer for PixivUtil worker queue processing interface. """ +from typing import Any, Protocol, cast + +from celery.result import AsyncResult from pixivutil_server_common.models import ( TagMetadataFilterMode, TagSortOrder, @@ -10,6 +13,14 @@ from pydantic import BaseModel +class CeleryTask(Protocol): + def apply_async(self, *, args: list[Any] | None = None, priority: int | None = None, **kwargs: Any) -> AsyncResult: ... + + +def as_celery_task(task: Any) -> CeleryTask: + return cast(CeleryTask, task) + + class DownloadArtworkByIdRequest(BaseModel): artwork_id: int diff --git a/PixivServer/routers/download_queue.py b/PixivServer/routers/download_queue.py index 0ec6cd5..e9f20be 100644 --- a/PixivServer/routers/download_queue.py +++ b/PixivServer/routers/download_queue.py @@ -19,15 +19,16 @@ from PixivServer.repository.pixivutil import PixivUtilRepository from PixivServer.utils import is_valid_date from PixivServer.worker.download import ( - delete_artwork_by_id, - download_artworks_by_id, - download_artworks_by_member_id, - download_artworks_by_tag, + delete_artwork_by_id_task, + download_artworks_by_id_task, + download_artworks_by_member_id_task, + download_artworks_by_tag_task, ) logger = logging.getLogger('uvicorn.pixivutil') router = APIRouter() + def get_artwork_and_member_name_from_db(artwork_id: int) -> tuple[str | None, str | None]: repository = PixivUtilRepository() try: @@ -69,7 +70,7 @@ async def queue_download_artwork_by_id( logger.info(f"Downloading Pixiv artwork by image ID: {artwork_id}.") request = DownloadArtworkByIdRequest(artwork_id=int(artwork_id)) artwork_title, member_name = get_artwork_and_member_name_from_db(request.artwork_id) - task: AsyncResult = download_artworks_by_id.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = download_artworks_by_id_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ "task_id": task.id, 'artwork_id': artwork_id, @@ -88,7 +89,7 @@ async def queue_download_artworks_by_member_id( logger.info(f"Downloading Pixiv artworks by member ID: {member_id}.") request = DownloadArtworksByMemberIdRequest(member_id=int(member_id)) member_name = get_member_name_from_db(request.member_id) - task: AsyncResult = download_artworks_by_member_id.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = download_artworks_by_member_id_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ "task_id": task.id, 'member_id': member_id, @@ -142,7 +143,7 @@ async def queue_download_artworks_by_tag( start_date=start_date, end_date=end_date, ) - task: AsyncResult = download_artworks_by_tag.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = download_artworks_by_tag_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ 'task_id': task.id, 'tag': decoded_tag, @@ -166,7 +167,7 @@ async def queue_delete_artwork_by_id( """ logger.info(f"Deleting Pixiv artwork by image ID: {artwork_id} (delete_metadata={delete_metadata}).") request = DeleteArtworkByIdRequest(artwork_id=int(artwork_id), delete_metadata=delete_metadata) - task: AsyncResult = delete_artwork_by_id.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = delete_artwork_by_id_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({ "task_id": task.id, 'artwork_id': artwork_id, diff --git a/PixivServer/routers/metadata_queue.py b/PixivServer/routers/metadata_queue.py index 04c026e..8472f2b 100644 --- a/PixivServer/routers/metadata_queue.py +++ b/PixivServer/routers/metadata_queue.py @@ -14,10 +14,10 @@ DownloadTagMetadataByIdRequest, ) from PixivServer.worker.metadata import ( - download_artwork_metadata_by_id, - download_member_metadata_by_id, - download_series_metadata_by_id, - download_tag_metadata_by_id, + download_artwork_metadata_by_id_task, + download_member_metadata_by_id_task, + download_series_metadata_by_id_task, + download_tag_metadata_by_id_task, ) logger = logging.getLogger("uvicorn.pixivutil") @@ -40,7 +40,7 @@ async def queue_download_member_metadata_by_id( member_id_int = int(member_id) logger.info(f"Queueing member metadata download by ID: {member_id_int}.") request = DownloadMemberMetadataByIdRequest(member_id=member_id_int) - task: AsyncResult = download_member_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = download_member_metadata_by_id_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({"task_id": task.id, "member_id": member_id_int}) @@ -60,7 +60,7 @@ async def queue_download_artwork_metadata_by_id( artwork_id_int = int(artwork_id) logger.info(f"Queueing artwork metadata download by ID: {artwork_id_int}.") request = DownloadArtworkMetadataByIdRequest(artwork_id=artwork_id_int) - task: AsyncResult = download_artwork_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = download_artwork_metadata_by_id_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({"task_id": task.id, "artwork_id": artwork_id_int}) @@ -80,7 +80,7 @@ async def queue_download_series_metadata_by_id( series_id_int = int(series_id) logger.info(f"Queueing series metadata download by ID: {series_id_int}.") request = DownloadSeriesMetadataByIdRequest(series_id=series_id_int) - task: AsyncResult = download_series_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = download_series_metadata_by_id_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse({"task_id": task.id, "series_id": series_id_int}) @@ -100,7 +100,7 @@ async def queue_download_tag_metadata_by_id( request = DownloadTagMetadataByIdRequest( tag=decoded_tag, filter_mode=filter_mode ) - task: AsyncResult = download_tag_metadata_by_id.apply_async(args=[request.model_dump()], priority=priority) + task: AsyncResult = download_tag_metadata_by_id_task.apply_async(args=[request.model_dump()], priority=priority) return JSONResponse( {"task_id": task.id, "tag": decoded_tag, "filter_mode": request.filter_mode} ) diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py index 4fc8ff8..62412f9 100644 --- a/PixivServer/worker/download.py +++ b/PixivServer/worker/download.py @@ -13,6 +13,7 @@ DownloadArtworkByIdRequest, DownloadArtworksByMemberIdRequest, DownloadArtworksByTagsRequest, + as_celery_task, ) from PixivServer.service.pixiv import PixivException @@ -99,3 +100,9 @@ def delete_artwork_by_id(self, request_dict: dict): return False finally: __job_sleep() + + +download_artworks_by_id_task = as_celery_task(download_artworks_by_id) +download_artworks_by_member_id_task = as_celery_task(download_artworks_by_member_id) +download_artworks_by_tag_task = as_celery_task(download_artworks_by_tag) +delete_artwork_by_id_task = as_celery_task(delete_artwork_by_id) diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py index 3fa2bb8..857a7d7 100644 --- a/PixivServer/worker/metadata.py +++ b/PixivServer/worker/metadata.py @@ -13,6 +13,7 @@ DownloadMemberMetadataByIdRequest, DownloadSeriesMetadataByIdRequest, DownloadTagMetadataByIdRequest, + as_celery_task, ) from PixivServer.service.pixiv import PixivException @@ -111,3 +112,9 @@ def download_tag_metadata_by_id(self, request_dict: dict): return False finally: __job_sleep() + + +download_member_metadata_by_id_task = as_celery_task(download_member_metadata_by_id) +download_artwork_metadata_by_id_task = as_celery_task(download_artwork_metadata_by_id) +download_series_metadata_by_id_task = as_celery_task(download_series_metadata_by_id) +download_tag_metadata_by_id_task = as_celery_task(download_tag_metadata_by_id) From a58fd8ee498e1097c3f1d821392dab5f9f08cf29 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 25 Feb 2026 00:20:23 -0800 Subject: [PATCH 21/32] address some pyright issues --- PixivServer/repository/pixivutil.py | 10 ++- PixivServer/repository/subscription.py | 94 +++++++++++++++----------- PixivServer/routers/dlq.py | 6 +- PixivServer/service/metrics.py | 2 +- 4 files changed, 70 insertions(+), 42 deletions(-) diff --git a/PixivServer/repository/pixivutil.py b/PixivServer/repository/pixivutil.py index 5f62715..144e06a 100644 --- a/PixivServer/repository/pixivutil.py +++ b/PixivServer/repository/pixivutil.py @@ -27,7 +27,7 @@ class PixivUtilRepository: def __init__(self): self.db_path = pixivutil_config.db_path - self.connection: sqlite3.Connection = None + self.connection: sqlite3.Connection = None # pyright: ignore[reportAttributeAccessIssue] this will be handled during open. def open(self): self.connection = sqlite3.connect(self.db_path, timeout=30.0) @@ -50,6 +50,7 @@ def get_member_data_by_id(self, member_id: int) -> PixivMemberPortfolio: Raises: KeyError: If member with the given ID is not found. """ + cursor = None try: cursor = self.connection.cursor() @@ -155,6 +156,7 @@ def get_all_pixiv_member_ids(self) -> list[int]: Returns: List of member IDs. Empty list if no members found. """ + cursor = None try: cursor = self.connection.cursor() cursor.execute("SELECT member_id FROM pixiv_master_member ORDER BY member_id ASC") @@ -174,6 +176,7 @@ def get_all_pixiv_image_ids(self) -> list[int]: Returns: List of image IDs. Empty list if no images found. """ + cursor = None try: cursor = self.connection.cursor() cursor.execute("SELECT image_id FROM pixiv_master_image ORDER BY image_id ASC") @@ -193,6 +196,7 @@ def get_all_pixiv_tags(self) -> list[str]: Returns: List of tag IDs. Empty list if no tags found. """ + cursor = None try: cursor = self.connection.cursor() cursor.execute("SELECT tag_id FROM pixiv_master_tag ORDER BY tag_id ASC") @@ -212,6 +216,7 @@ def get_all_pixiv_series(self) -> list[str]: Returns: List of series IDs. Empty list if no series found. """ + cursor = None try: cursor = self.connection.cursor() cursor.execute("SELECT series_id FROM pixiv_master_series ORDER BY series_id ASC") @@ -231,6 +236,7 @@ def get_tag_info_by_id(self, tag_id: str) -> PixivTagInfo: Raises: KeyError: If tag with the given ID is not found. """ + cursor = None try: cursor = self.connection.cursor() @@ -305,6 +311,7 @@ def get_series_info_by_id(self, series_id: str) -> PixivSeriesInfo: Raises: KeyError: If series with the given ID is not found. """ + cursor = None try: cursor = self.connection.cursor() @@ -365,6 +372,7 @@ def get_image_data_by_id(self, image_id: int) -> PixivImageComplete: Raises: KeyError: If image with the given ID is not found. """ + cursor = None try: cursor = self.connection.cursor() diff --git a/PixivServer/repository/subscription.py b/PixivServer/repository/subscription.py index ecc452a..7b977d4 100644 --- a/PixivServer/repository/subscription.py +++ b/PixivServer/repository/subscription.py @@ -9,7 +9,7 @@ class SubscriptionRepository: def __init__(self): self.db_path = pixivutil_config.db_path - self.connection: sqlite3.Connection = None + self.connection: sqlite3.Connection = None # pyright: ignore[reportAttributeAccessIssue] this will be handled during open. def open(self): self.connection = sqlite3.connect(self.db_path) @@ -35,57 +35,63 @@ def create_table(self): def check_member_id_exist(self, member_id: int) -> bool: result = False + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''SELECT 1 FROM pixiv_server_member_subscription WHERE member_id = ?''', (member_id, ) ) - result = c.fetchone() + result = cursor.fetchone() except Exception as e: logger.error(f'Failed to check existence of member ID: {member_id}') raise e finally: - c.close() + if cursor is not None: + cursor.close() return result - def select_member_name_by_id(self, member_id: int) -> str: - result: str = None + def select_member_name_by_id(self, member_id: int) -> str | None: + result: str | None = None + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''SELECT name FROM pixiv_server_member_subscription WHERE member_id = ?''', (member_id, ) ) - result = c.fetchone() + result = cursor.fetchone() except Exception as e: logger.error(f'Failed to retrieve member name for ID {member_id}.') raise e finally: - c.close() + if cursor is not None: + cursor.close() return result def select_member_subscriptions(self) -> list[tuple[int, str]]: - results: list = None - + results: list[tuple[int, str]] = [] + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''SELECT member_id, name FROM pixiv_server_member_subscription ORDER BY member_id''', ) - results = c.fetchall() + results = cursor.fetchall() except Exception as e: logger.error('Failed to export member subscriptions: ', e) raise e finally: - c.close() + if cursor is not None: + cursor.close() return results def add_member_subscription(self, member_id: int, member_name: str) -> bool: + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''INSERT OR IGNORE INTO pixiv_server_member_subscription VALUES(?, ?, datetime('now'), datetime('now'))''', (member_id, member_name, ) ) @@ -94,13 +100,15 @@ def add_member_subscription(self, member_id: int, member_name: str) -> bool: logger.error(f'Failed to add member subscription: {member_id}', e) raise e finally: - c.close() + if cursor is not None: + cursor.close() return True def remove_member_subscription(self, member_id: int) -> bool: + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''DELETE FROM pixiv_server_member_subscription WHERE member_id = ?''', (member_id, ) ) @@ -109,46 +117,51 @@ def remove_member_subscription(self, member_id: int) -> bool: logger.error(f"Failed to remove member {member_id} from subscription: ", e) raise e finally: - c.close() + if cursor is not None: + cursor.close() return True def check_tag_name_exist(self, tag_id: str) -> str: result = False + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''SELECT 1 FROM pixiv_server_tag_subscription WHERE tag_id = ?''', (tag_id, ) ) - result = c.fetchone() + result = cursor.fetchone() except Exception as e: logger.error(f"Failed to check existence of tag name: {tag_id}") raise e finally: - c.close() + if cursor is not None: + cursor.close() return result def select_tag_subscriptions(self) -> list[tuple[str]]: - results: list = None - + results: list[tuple[str]] = [] + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''SELECT tag_id FROM pixiv_server_tag_subscription''', ) - results = c.fetchall() + results = cursor.fetchall() except Exception as e: logger.error("Failed to export tag encoded subscriptions: ", e) raise e finally: - c.close() + if cursor is not None: + cursor.close() return results def add_tag_subscription(self, tag_id: str, bookmark_count: int) -> bool: + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''INSERT INTO pixiv_server_tag_subscription (tag_id, bookmark_count, created_date, last_modified_date) VALUES(?, ?, datetime('now'), datetime('now')) ON CONFLICT(tag_id) @@ -164,13 +177,15 @@ def add_tag_subscription(self, tag_id: str, bookmark_count: int) -> bool: logger.error(f'Failed to add encoded tag subscription: {tag_id}', e) raise e finally: - c.close() + if cursor is not None: + cursor.close() return True def remove_tag_subscription(self, tag_id: int) -> bool: + cursor = None try: - c = self.connection.cursor() - c.execute( + cursor = self.connection.cursor() + cursor.execute( '''DELETE FROM pixiv_server_tag_subscription WHERE tag_id = ?''', (tag_id, ) ) @@ -179,7 +194,8 @@ def remove_tag_subscription(self, tag_id: int) -> bool: logger.error(f"Failed to remove tag {tag_id} from subscription: ", e) raise e finally: - c.close() + if cursor is not None: + cursor.close() return True def close(self): diff --git a/PixivServer/routers/dlq.py b/PixivServer/routers/dlq.py index 19f7743..0a9c8b4 100644 --- a/PixivServer/routers/dlq.py +++ b/PixivServer/routers/dlq.py @@ -7,7 +7,11 @@ from kombu import Connection from PixivServer.config import rabbitmq -from PixivServer.config.celery import MAIN_ROUTING_KEY, dead_letter_queue, default_exchange +from PixivServer.config.celery import ( + MAIN_ROUTING_KEY, + dead_letter_queue, + default_exchange, +) from PixivServer.worker import pixiv_worker logger = logging.getLogger('uvicorn.pixivutil') diff --git a/PixivServer/service/metrics.py b/PixivServer/service/metrics.py index 8c10a71..b79ffbe 100644 --- a/PixivServer/service/metrics.py +++ b/PixivServer/service/metrics.py @@ -18,9 +18,9 @@ # import PixivServer.routers.subscription import PixivServer.service import PixivServer.service.pixiv +from PixivServer.config.celery import DEAD_LETTER_QUEUE_NAME, MAIN_QUEUE_NAME from PixivServer.config.pixivutil import config as pixivutil_config from PixivServer.config.rabbitmq import config as rabbitmq_config -from PixivServer.config.celery import DEAD_LETTER_QUEUE_NAME, MAIN_QUEUE_NAME from PixivServer.metrics import ( DB_ARTWORKS, DB_MEMBERS, From 9813445baabda23e3a971c1db301546a426c511e Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 25 Feb 2026 00:21:34 -0800 Subject: [PATCH 22/32] address ruff linting --- integration_tests/test_queue_priority.py | 8 ++++++-- tests/test_celery_config.py | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_queue_priority.py b/integration_tests/test_queue_priority.py index 52801a9..4b8ffc0 100644 --- a/integration_tests/test_queue_priority.py +++ b/integration_tests/test_queue_priority.py @@ -2,7 +2,11 @@ import pytest -from PixivServer.config.celery import LEGACY_MAIN_QUEUE_NAME, MAIN_QUEUE_NAME, QUEUE_MAX_PRIORITY +from PixivServer.config.celery import ( + LEGACY_MAIN_QUEUE_NAME, + MAIN_QUEUE_NAME, + QUEUE_MAX_PRIORITY, +) @pytest.mark.pixiv_api @@ -87,7 +91,7 @@ def test_priority_values_above_queue_max_are_not_a_higher_tier_than_max(clean_en over_limit = QUEUE_MAX_PRIORITY + 1 clean_env.api_json("POST", "/api/dev/priority/L1?priority=1&sleep_ms=1800") clean_env.api_json("POST", "/api/dev/priority/H3?priority=3&sleep_ms=200") - clean_env.api_json(f"POST", f"/api/dev/priority/HOver?priority={over_limit}&sleep_ms=200") + clean_env.api_json("POST", f"/api/dev/priority/HOver?priority={over_limit}&sleep_ms=200") clean_env.api_json("POST", "/api/dev/priority/N1?priority=2&sleep_ms=200") state = clean_env.wait_for_priority_probe_started_count(4, timeout=90) diff --git a/tests/test_celery_config.py b/tests/test_celery_config.py index b3edee5..7fd2b4b 100644 --- a/tests/test_celery_config.py +++ b/tests/test_celery_config.py @@ -1,6 +1,10 @@ from celery import Celery -from PixivServer.config.celery import DLX_EXCHANGE_NAME, MAIN_QUEUE_NAME, QUEUE_MAX_PRIORITY +from PixivServer.config.celery import ( + DLX_EXCHANGE_NAME, + MAIN_QUEUE_NAME, + QUEUE_MAX_PRIORITY, +) def test_celery_failure_is_rejected_for_rabbitmq_dlq(): From 21a48cfe3a86289bd819eaf911cec5fcd18d58ae Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:14:23 -0800 Subject: [PATCH 23/32] dlq logic --- PixivServer/service/pixiv.py | 63 +++++++++++++++++++++++++++++++++- PixivServer/worker/download.py | 11 ++++-- PixivServer/worker/metadata.py | 11 ++++-- docker-compose.yml | 8 +++++ integration_tests/conftest.py | 63 ++++++++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 7 deletions(-) diff --git a/PixivServer/service/pixiv.py b/PixivServer/service/pixiv.py index d8aba8e..e46a122 100644 --- a/PixivServer/service/pixiv.py +++ b/PixivServer/service/pixiv.py @@ -23,6 +23,7 @@ PixivArtistHandler, PixivBrowserFactory, PixivConfig, + PixivConstant, PixivDBManager, PixivException, PixivHelper, @@ -90,6 +91,12 @@ def load_environment_variables(self): if pixivutil_config.cookie: __config__.cookie = pixivutil_config.cookie + pixiv_retry = os.getenv("PIXIVUTIL2_NETWORK_RETRY") + if pixiv_retry is not None: + __config__.retry = int(pixiv_retry) + pixiv_retry_wait = os.getenv("PIXIVUTIL2_NETWORK_RETRY_WAIT") + if pixiv_retry_wait is not None: + __config__.retryWait = int(pixiv_retry_wait) return @@ -192,6 +199,52 @@ def get_artwork_name(self, artwork_id: int) -> str: raise PixivException("Cannot get artwork name; response: " + str(response)) return data.imageTitle + def _raise_metadata_process_image_failure( + self, + *, + artwork_id: int, + process_result: int, + previous_error_list_len: int, + previous_error_code: int, + ) -> None: + if process_result == PixivConstant.PIXIVUTIL_OK: + return + + error_list = globals()["__errorList"] + new_errors = error_list[previous_error_list_len:] if len(error_list) >= previous_error_list_len else error_list + pixiv_error: PixivException | None = None + for item in reversed(new_errors): + if not isinstance(item, dict): + continue + exc = item.get("exception") + if isinstance(exc, PixivException): + pixiv_error = exc + break + + if pixiv_error is not None: + if pixiv_error.errorCode in (PixivException.DOWNLOAD_FAILED_NETWORK, PixivException.SERVER_ERROR): + raise ConnectionError( + f"Artwork metadata fetch failed due to network/server error for artwork_id={artwork_id}" + ) from pixiv_error + raise RuntimeError( + f"Artwork metadata fetch failed for artwork_id={artwork_id} " + f"(pixiv_error_code={pixiv_error.errorCode}, result={process_result})" + ) from pixiv_error + + current_error_code = ERROR_CODE + error_code = current_error_code if current_error_code != previous_error_code else -1 + if error_code in (PixivException.DOWNLOAD_FAILED_NETWORK, PixivException.SERVER_ERROR): + raise ConnectionError( + f"Artwork metadata fetch failed due to network/server error for artwork_id={artwork_id} " + f"(pixiv_error_code={error_code}, result={process_result})" + ) + # Metadata fetch failures can be swallowed by PixivUtil2 without a stable error code. + # Treat unclassified failures as transient so worker retry/DLQ policy can recover them. + raise ConnectionError( + f"Artwork metadata fetch failed with unclassified error for artwork_id={artwork_id} " + f"(pixiv_error_code={error_code}, result={process_result})" + ) + def download_artwork_by_id(self, request: DownloadArtworkByIdRequest): PixivHelper.print_and_log("info", f"Download by artwork ID: {request.artwork_id}") return PixivImageHandler.process_image( @@ -328,7 +381,9 @@ def download_member_metadata_by_id(self, request: DownloadMemberMetadataByIdRequ def download_artwork_metadata_by_id(self, request: DownloadArtworkMetadataByIdRequest): PixivHelper.print_and_log("info", f"Download artwork metadata by ID: {request.artwork_id}") - PixivImageHandler.process_image( + previous_error_list_len = len(globals()["__errorList"]) + previous_error_code = ERROR_CODE + result = PixivImageHandler.process_image( sys.modules[__name__], __config__, artist=None, @@ -336,6 +391,12 @@ def download_artwork_metadata_by_id(self, request: DownloadArtworkMetadataByIdRe useblacklist=False, metadata_only=True, ) + self._raise_metadata_process_image_failure( + artwork_id=request.artwork_id, + process_result=result, + previous_error_list_len=previous_error_list_len, + previous_error_code=previous_error_code, + ) def download_series_metadata_by_id(self, request: DownloadSeriesMetadataByIdRequest): PixivHelper.print_and_log("info", f"Download series metadata by ID: {request.series_id}") diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py index 62412f9..e551b56 100644 --- a/PixivServer/worker/download.py +++ b/PixivServer/worker/download.py @@ -1,4 +1,5 @@ import logging +import os import random import time import traceback @@ -19,12 +20,16 @@ logger = logging.getLogger(__name__) -_NETWORK_MAX_RETRIES = 3 -_NETWORK_RETRY_COUNTDOWN = 60 +_NETWORK_MAX_RETRIES = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES", "3")) +_NETWORK_RETRY_COUNTDOWN = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN", "60")) +_JOB_SLEEP_MIN_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS", "1000")) +_JOB_SLEEP_MAX_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS", "5000")) def __job_sleep(): - time.sleep(random.uniform(1, 5)) + lower_ms = min(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) + upper_ms = max(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) + time.sleep(random.uniform(lower_ms, upper_ms) / 1000) return 0 diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py index 857a7d7..7b95523 100644 --- a/PixivServer/worker/metadata.py +++ b/PixivServer/worker/metadata.py @@ -1,4 +1,5 @@ import logging +import os import random import time import traceback @@ -19,12 +20,16 @@ logger = logging.getLogger(__name__) -_NETWORK_MAX_RETRIES = 3 -_NETWORK_RETRY_COUNTDOWN = 60 +_NETWORK_MAX_RETRIES = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES", "3")) +_NETWORK_RETRY_COUNTDOWN = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN", "60")) +_JOB_SLEEP_MIN_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS", "1000")) +_JOB_SLEEP_MAX_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS", "5000")) def __job_sleep(): - time.sleep(random.uniform(1, 5)) + lower_ms = min(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) + upper_ms = max(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) + time.sleep(random.uniform(lower_ms, upper_ms) / 1000) return 0 diff --git a/docker-compose.yml b/docker-compose.yml index df14b52..99f2b82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,12 @@ services: - PIXIVUTIL_COOKIE=$PIXIVUTIL_COOKIE - PIXIVUTIL_SERVER_ENV=development - RABBITMQ_BROKER_URL=amqp://guest:guest@rabbitmq:5672 + - PIXIVUTIL2_NETWORK_RETRY=${PIXIVUTIL2_NETWORK_RETRY:-3} + - PIXIVUTIL2_NETWORK_RETRY_WAIT=${PIXIVUTIL2_NETWORK_RETRY_WAIT:-5} + - PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES=${PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES:-3} + - PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN=${PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN:-60} + - PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS=${PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS:-1000} + - PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS=${PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS:-5000} volumes: - pixivutil-data:/workdir/.pixivUtil2 - pixivutil-downloads:/workdir/downloads @@ -54,6 +60,8 @@ services: - PIXIVUTIL_SERVER_API_KEY=$PIXIVUTIL_SERVER_API_KEY - PIXIVUTIL_SERVER_ENV=development - RABBITMQ_BROKER_URL=amqp://guest:guest@rabbitmq:5672 + - PIXIVUTIL2_NETWORK_RETRY=${PIXIVUTIL2_NETWORK_RETRY:-3} + - PIXIVUTIL2_NETWORK_RETRY_WAIT=${PIXIVUTIL2_NETWORK_RETRY_WAIT:-5} volumes: - pixivutil-data:/workdir/.pixivUtil2 - pixivutil-downloads:/workdir/downloads diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index bd5368a..05cd2dd 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -1,4 +1,5 @@ import json +import os import shutil import subprocess import time @@ -148,6 +149,62 @@ def wait_for_queue_count(self, queue_name: str, expected: int, timeout: int = 45 f"Queue {queue_name!r} did not reach {expected} within {timeout}s (last={last_count})" ) + def wait_for_dead_letter_message(self, dead_letter_id: str, timeout: int = 90) -> dict: + deadline = time.time() + timeout + last_messages: list[dict] | None = None + while time.time() < deadline: + messages = self.api_json("GET", "/api/queue/dead-letter/") + if not isinstance(messages, list): + raise RuntimeError(f"Unexpected DLQ payload: {messages!r}") + last_messages = messages + for message in messages: + if isinstance(message, dict) and message.get("dead_letter_id") == dead_letter_id: + return message + time.sleep(1) + queue_counts: dict[str, int] | str + worker_logs: str + try: + queue_counts = self.rabbitmq_queue_counts() + except Exception as exc: # noqa: BLE001 + queue_counts = f"" + try: + worker_logs = self.compose("logs", "--tail=200", "pixivutil-worker", timeout=90).stdout + except Exception as exc: # noqa: BLE001 + worker_logs = f"" + raise RuntimeError( + f"DLQ message {dead_letter_id!r} not found within {timeout}s " + f"(last_messages={last_messages}, queue_counts={queue_counts}, worker_logs_tail={worker_logs})" + ) + + def block_worker_pixiv_hosts(self) -> None: + self.docker_exec( + "pixivutil-worker", + "sh", + "-lc", + ( + "printf '\\n0.0.0.0 www.pixiv.net # codex-inttest\\n" + "0.0.0.0 i.pximg.net # codex-inttest\\n' >> /etc/hosts" + ), + timeout=30, + ) + + def unblock_worker_pixiv_hosts(self) -> None: + script = ( + "from pathlib import Path\n" + "path = Path('/etc/hosts')\n" + "lines = path.read_text().splitlines()\n" + "filtered = [line for line in lines if 'codex-inttest' not in line]\n" + "path.write_text('\\n'.join(filtered) + ('\\n' if filtered else ''))\n" + ) + self.docker_exec( + "pixivutil-worker", + "python", + "-c", + script, + check=False, + timeout=30, + ) + def wait_worker_log_contains(self, needle: str, timeout: int = 45) -> None: deadline = time.time() + timeout while time.time() < deadline: @@ -198,6 +255,7 @@ def wait_for_priority_probe_started_count(self, expected: int, timeout: int = 90 ) def clear_state(self) -> None: + self.unblock_worker_pixiv_hosts() self.docker_exec( "pixivutil-worker", "sh", @@ -240,6 +298,11 @@ def clear_state(self) -> None: @pytest.fixture(scope="session") def compose_env() -> ComposeTestEnv: _require_docker_tools() + os.environ.setdefault("PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN", "1") + os.environ.setdefault("PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS", "10") + os.environ.setdefault("PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS", "50") + os.environ.setdefault("PIXIVUTIL2_NETWORK_RETRY", "0") + os.environ.setdefault("PIXIVUTIL2_NETWORK_RETRY_WAIT", "1") env = ComposeTestEnv() env.compose("down", "--volumes", check=False, timeout=180) env.compose("up", "--build", "-d", timeout=600) From 2efcaf9bda64d20622219adac043fd5627b6656c Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:19:46 -0800 Subject: [PATCH 24/32] include protocol for PixivConfig --- PixivServer/service/pixiv.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/PixivServer/service/pixiv.py b/PixivServer/service/pixiv.py index e46a122..bba9e5d 100644 --- a/PixivServer/service/pixiv.py +++ b/PixivServer/service/pixiv.py @@ -3,6 +3,7 @@ import sqlite3 import sys import traceback +from typing import Protocol, cast from urllib.error import HTTPError sys.path.append('PixivUtil2') @@ -34,12 +35,25 @@ logger = logging.getLogger(__name__) + +class PixivConfigProtocol(Protocol): + """Structural protocol for the PixivConfig attributes used by PixivUtilService.""" + cookie: str + retry: int + retryWait: int + dbPath: str + rootDirectory: str + + def loadConfig(self, path: str | None = None) -> None: ... + def writeConfig(self, error: bool = False, path: str | None = None) -> None: ... + + # ------ START CALLER ITEMS ------ -__config__ = PixivConfig.PixivConfig() +__config__: PixivConfigProtocol = cast(PixivConfigProtocol, PixivConfig.PixivConfig()) configfile = ".pixivUtil2/conf/config.ini" -__dbManager__ = None -__br__: PixivBrowserFactory.PixivBrowser = None +__dbManager__: PixivDBManager | None = None +__br__: PixivBrowserFactory.PixivBrowser | None = None __blacklistTags = [] __suppressTags = [] __log__ = None @@ -134,6 +148,7 @@ def close(self): PixivHelper.print_and_log("info", "Closing...") # self.remove_database() __config__.writeConfig(path=configfile) + assert __dbManager__ is not None __dbManager__.close() def open_database(self): @@ -153,6 +168,7 @@ def configure_database_connection(self, connection: sqlite3.Connection) -> None: cursor.close() def remove_database(self): + assert __dbManager__ is not None __dbManager__.close() os.remove(__config__.dbPath) @@ -166,6 +182,7 @@ def reset_downloads(self): def login_pixiv(self, cookie) -> bool: result = False try: + assert __br__ is not None result = __br__.loginUsingCookie(login_cookie=cookie) except (HTTPError, PixivException, AssertionError, ValueError) as e: logger.error(f'Error at doLogin(): {sys.exc_info()}') From ee25b031b703f8058a5a1fe802776cd724986dc0 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:30:17 -0800 Subject: [PATCH 25/32] add dlq documentation --- README.md | 5 +++++ docs/api/dlq.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 docs/api/dlq.md diff --git a/README.md b/README.md index 8b69ae3..249a154 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,11 @@ For example, the server supports the following endpoints: - Get tag metadata by ID - Get series metadata by ID +#### [Dead letter queue (DLQ)](/docs/api/dlq.md) + +API endpoints to inspect, replay, and purge failed worker messages in the dead +letter queue. + #### [Download queueing](/docs/api/download.md) API endpoints to queue content (artwork) downloads for worker from server. diff --git a/docs/api/dlq.md b/docs/api/dlq.md new file mode 100644 index 0000000..1602a4c --- /dev/null +++ b/docs/api/dlq.md @@ -0,0 +1,59 @@ +# Dead Letter Queue API + +Authentication: +- Requires `Authorization: Bearer ` when `PIXIVUTIL_SERVER_API_KEY` is set. +- If `PIXIVUTIL_SERVER_API_KEY` is unset/empty, authentication is disabled. + +Dead letter queue (DLQ) endpoints manage failed worker messages that were moved +to the broker dead letter queue after retry exhaustion or terminal failure. + +`GET /api/queue/dead-letter/` + +List all messages currently in the dead letter queue. + +Response item shape: +- `dead_letter_id`: message/task identifier (string) +- `task_name`: registered Celery task name (string) +- `payload`: original task payload (object) + +`POST /api/queue/dead-letter/resume` + +Requeue all resumable dead letter messages back to the main worker queue. + +Response: +- `requeued`: number of messages requeued + +Notes: +- Messages with unknown/unregistered task names are left in the DLQ. +- Unparseable messages are left in the DLQ. + +`POST /api/queue/dead-letter/{dead_letter_id}/resume` + +Requeue a specific dead letter message by `dead_letter_id`. + +Response: +- `dead_letter_id`: requested DLQ message id +- `requeued`: `true` when message was requeued +- `task_name`: task name that was requeued + +Errors: +- `404`: dead letter message not found +- `422`: task name is not recognized and cannot be resumed + +`DELETE /api/queue/dead-letter/` + +Purge all messages from the dead letter queue. + +Response: +- `dropped`: number of messages removed + +`DELETE /api/queue/dead-letter/{dead_letter_id}` + +Drop a specific dead letter message by `dead_letter_id`. + +Response: +- `dead_letter_id`: requested DLQ message id +- `dropped`: `true` when message was removed + +Errors: +- `404`: dead letter message not found From 00eef6ddead40dadc8e31d56c2778e8a3586c4e4 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:43:30 -0800 Subject: [PATCH 26/32] disable rabbitmq timeout --- rabbitmq/rabbitmq.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rabbitmq/rabbitmq.conf b/rabbitmq/rabbitmq.conf index 9de9156..659e406 100644 --- a/rabbitmq/rabbitmq.conf +++ b/rabbitmq/rabbitmq.conf @@ -1,2 +1,3 @@ -# 24 hour timeout. -consumer_timeout = 86400000 \ No newline at end of file +# Disable consumer delivery acknowledgement timeout to prevent +# long-running tasks crashing worker. +consumer_timeout = 0 \ No newline at end of file From 5e65997bc9695f1bad91c91cd1183f0d1ab5085f Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Fri, 6 Mar 2026 02:26:32 -0800 Subject: [PATCH 27/32] Revert "disable rabbitmq timeout" This reverts commit 00eef6ddead40dadc8e31d56c2778e8a3586c4e4. --- rabbitmq/rabbitmq.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rabbitmq/rabbitmq.conf b/rabbitmq/rabbitmq.conf index 659e406..9de9156 100644 --- a/rabbitmq/rabbitmq.conf +++ b/rabbitmq/rabbitmq.conf @@ -1,3 +1,2 @@ -# Disable consumer delivery acknowledgement timeout to prevent -# long-running tasks crashing worker. -consumer_timeout = 0 \ No newline at end of file +# 24 hour timeout. +consumer_timeout = 86400000 \ No newline at end of file From 7cf4b4765e0b056317d77a5ad16b04c6f9bca5a0 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sun, 8 Mar 2026 15:05:15 -0700 Subject: [PATCH 28/32] remove integration tests --- integration_tests/README.md | 9 - integration_tests/__init__.py | 1 - integration_tests/conftest.py | 320 ----------------------- integration_tests/test_dlq_replay.py | 153 ----------- integration_tests/test_queue_priority.py | 131 ---------- pyproject.toml | 2 +- 6 files changed, 1 insertion(+), 615 deletions(-) delete mode 100644 integration_tests/README.md delete mode 100644 integration_tests/__init__.py delete mode 100644 integration_tests/conftest.py delete mode 100644 integration_tests/test_dlq_replay.py delete mode 100644 integration_tests/test_queue_priority.py diff --git a/integration_tests/README.md b/integration_tests/README.md deleted file mode 100644 index 92d2beb..0000000 --- a/integration_tests/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# PixivUtil Server Integration Test Suite - -Run integration tests: - -```sh -uv run pytest integration_tests -``` - -Docker and Docker Compose are required. diff --git a/integration_tests/__init__.py b/integration_tests/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/integration_tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py deleted file mode 100644 index 05cd2dd..0000000 --- a/integration_tests/conftest.py +++ /dev/null @@ -1,320 +0,0 @@ -import json -import os -import shutil -import subprocess -import time -import urllib.error -import urllib.request -from pathlib import Path - -import pytest - -from PixivServer.config.celery import ( - DEAD_LETTER_QUEUE_NAME, - LEGACY_MAIN_QUEUE_NAME, - MAIN_QUEUE_NAME, -) - -REPO_ROOT = Path(__file__).resolve().parents[1] -BASE_URL = "http://localhost:8000" -AUTH_HEADER = ("Authorization", "Bearer pixiv") -DEV_SUCCESS_SENTINEL = "/tmp/pixivutil-dev-dlq-success.flag" # TODO: this requires a redesign. - -def _run( - args: list[str], - *, - check: bool = True, - timeout: int = 120, - cwd: Path = REPO_ROOT, -) -> subprocess.CompletedProcess[str]: - try: - proc = subprocess.run( - args, - cwd=cwd, - capture_output=True, - text=True, - timeout=timeout, - check=False, - ) - except FileNotFoundError as exc: - raise RuntimeError(f"Command not found: {args[0]}") from exc - except subprocess.TimeoutExpired as exc: - raise RuntimeError(f"Command timed out ({timeout}s): {' '.join(args)}") from exc - - if check and proc.returncode != 0: - stderr = proc.stderr.strip() - stdout = proc.stdout.strip() - detail = stderr or stdout - raise RuntimeError(f"Command failed ({proc.returncode}): {' '.join(args)}\n{detail}") - return proc - - -def _require_docker_tools() -> None: - if shutil.which("docker") is None: - pytest.exit("docker is not available in PATH", returncode=1) - _run(["docker", "--version"], timeout=15) - _run(["docker", "compose", "version"], timeout=15) - - -def _http_json(method: str, path: str) -> object: - req = urllib.request.Request(f"{BASE_URL}{path}", method=method) - req.add_header(*AUTH_HEADER) - with urllib.request.urlopen(req, timeout=10) as resp: - return json.loads(resp.read()) - - -class ComposeTestEnv: - def compose(self, *args: str, check: bool = True, timeout: int = 240) -> subprocess.CompletedProcess[str]: - return _run(["docker", "compose", *args], check=check, timeout=timeout) - - def docker_exec(self, container: str, *args: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess[str]: - return _run(["docker", "exec", container, *args], check=check, timeout=timeout) - - def wait_http_ready(self, timeout: int = 60) -> None: - deadline = time.time() + timeout - last_error: str | None = None - while time.time() < deadline: - try: - req = urllib.request.Request(f"{BASE_URL}/", method="GET") - with urllib.request.urlopen(req, timeout=2): - return - except Exception as exc: # noqa: BLE001 - last_error = str(exc) - time.sleep(1) - raise RuntimeError(f"Server did not become ready within {timeout}s: {last_error}") - - def api_json(self, method: str, path: str) -> object: - return _http_json(method, path) - - def dev_task_state(self, task_id: str) -> dict: - state = self.api_json("GET", f"/api/dev/task/{task_id}") - if not isinstance(state, dict): - raise RuntimeError(f"Unexpected dev task state payload for {task_id}: {state!r}") - return state - - def rabbitmq_queue_counts(self) -> dict[str, int]: - proc = self.docker_exec("rabbitmq", "rabbitmqctl", "list_queues", "name", "messages", timeout=90) - counts: dict[str, int] = {} - for line in proc.stdout.splitlines(): - if "\t" not in line: - continue - name, count = line.split("\t", 1) - if name == "name": - continue - try: - counts[name.strip()] = int(count.strip()) - except ValueError: - continue - return counts - - def restart_worker(self) -> None: - self.compose("restart", "pixivutil-worker", timeout=240) - - def wait_for_queue_absent(self, queue_name: str, timeout: int = 45) -> None: - deadline = time.time() + timeout - last_counts: dict[str, int] | None = None - while time.time() < deadline: - counts = self.rabbitmq_queue_counts() - last_counts = counts - if queue_name not in counts: - return - time.sleep(1) - raise RuntimeError(f"Queue {queue_name!r} still exists after {timeout}s (last_counts={last_counts})") - - def seed_legacy_main_queue_message(self, body: str = "legacy-cutover-test") -> None: - script = ( - "from kombu import Connection, Exchange, Queue; " - f"from PixivServer.config.celery import LEGACY_MAIN_EXCHANGE_NAME, LEGACY_MAIN_QUEUE_NAME; " - "conn = Connection('amqp://guest:guest@rabbitmq:5672'); " - "conn.connect(); " - "ex = Exchange(LEGACY_MAIN_EXCHANGE_NAME, type='direct', durable=True); " - "q = Queue(LEGACY_MAIN_QUEUE_NAME, exchange=ex, routing_key=LEGACY_MAIN_QUEUE_NAME, durable=True); " - "q = q.bind(conn); q.declare(); " - "producer = conn.Producer(); " - f"producer.publish({body!r}, exchange=ex, routing_key=LEGACY_MAIN_QUEUE_NAME, " - "content_type='text/plain', content_encoding='utf-8', delivery_mode=2); " - "conn.release()" - ) - self.docker_exec("pixivutil-worker", "python", "-c", script, timeout=120) - - def wait_for_queue_count(self, queue_name: str, expected: int, timeout: int = 45) -> None: - deadline = time.time() + timeout - last_count: int | None = None - while time.time() < deadline: - last_count = self.rabbitmq_queue_counts().get(queue_name) - if last_count == expected: - return - time.sleep(1) - raise RuntimeError( - f"Queue {queue_name!r} did not reach {expected} within {timeout}s (last={last_count})" - ) - - def wait_for_dead_letter_message(self, dead_letter_id: str, timeout: int = 90) -> dict: - deadline = time.time() + timeout - last_messages: list[dict] | None = None - while time.time() < deadline: - messages = self.api_json("GET", "/api/queue/dead-letter/") - if not isinstance(messages, list): - raise RuntimeError(f"Unexpected DLQ payload: {messages!r}") - last_messages = messages - for message in messages: - if isinstance(message, dict) and message.get("dead_letter_id") == dead_letter_id: - return message - time.sleep(1) - queue_counts: dict[str, int] | str - worker_logs: str - try: - queue_counts = self.rabbitmq_queue_counts() - except Exception as exc: # noqa: BLE001 - queue_counts = f"" - try: - worker_logs = self.compose("logs", "--tail=200", "pixivutil-worker", timeout=90).stdout - except Exception as exc: # noqa: BLE001 - worker_logs = f"" - raise RuntimeError( - f"DLQ message {dead_letter_id!r} not found within {timeout}s " - f"(last_messages={last_messages}, queue_counts={queue_counts}, worker_logs_tail={worker_logs})" - ) - - def block_worker_pixiv_hosts(self) -> None: - self.docker_exec( - "pixivutil-worker", - "sh", - "-lc", - ( - "printf '\\n0.0.0.0 www.pixiv.net # codex-inttest\\n" - "0.0.0.0 i.pximg.net # codex-inttest\\n' >> /etc/hosts" - ), - timeout=30, - ) - - def unblock_worker_pixiv_hosts(self) -> None: - script = ( - "from pathlib import Path\n" - "path = Path('/etc/hosts')\n" - "lines = path.read_text().splitlines()\n" - "filtered = [line for line in lines if 'codex-inttest' not in line]\n" - "path.write_text('\\n'.join(filtered) + ('\\n' if filtered else ''))\n" - ) - self.docker_exec( - "pixivutil-worker", - "python", - "-c", - script, - check=False, - timeout=30, - ) - - def wait_worker_log_contains(self, needle: str, timeout: int = 45) -> None: - deadline = time.time() + timeout - while time.time() < deadline: - logs = self.compose("logs", "--tail=400", "pixivutil-worker", timeout=90).stdout - if needle in logs: - return - time.sleep(1) - raise RuntimeError(f"Worker logs did not contain expected text within {timeout}s: {needle}") - - def wait_for_dev_task_terminal_state(self, task_id: str, terminal_state: str, timeout: int = 45) -> dict: - deadline = time.time() + timeout - last_state: dict | None = None - last_error: str | None = None - while time.time() < deadline: - try: - state = self.dev_task_state(task_id) - last_state = state - if state.get("terminal_state") == terminal_state: - return state - except urllib.error.HTTPError as exc: - last_error = str(exc) - if exc.code != 404: - raise - time.sleep(1) - raise RuntimeError( - f"Dev task {task_id!r} did not reach terminal_state={terminal_state!r} within {timeout}s " - f"(last_state={last_state}, last_error={last_error})" - ) - - def dev_priority_probe_state(self) -> dict: - state = self.api_json("GET", "/api/dev/priority") - if not isinstance(state, dict): - raise RuntimeError(f"Unexpected dev priority probe payload: {state!r}") - return state - - def wait_for_priority_probe_started_count(self, expected: int, timeout: int = 90) -> dict: - deadline = time.time() + timeout - last_state: dict | None = None - while time.time() < deadline: - state = self.dev_priority_probe_state() - last_state = state - started = state.get("started") - if isinstance(started, list) and len(started) >= expected: - return state - time.sleep(0.5) - raise RuntimeError( - f"Priority probe did not reach started_count>={expected} within {timeout}s (last_state={last_state})" - ) - - def clear_state(self) -> None: - self.unblock_worker_pixiv_hosts() - self.docker_exec( - "pixivutil-worker", - "sh", - "-lc", - ( - f"rm -f {DEV_SUCCESS_SENTINEL} " - "/workdir/.pixivUtil2/dev-dlq-task-state.json " - "/workdir/.pixivUtil2/dev-dlq-task-state.tmp " - "/workdir/.pixivUtil2/dev-priority-task-state.json " - "/workdir/.pixivUtil2/dev-priority-task-state.tmp" - ), - check=False, - ) - self.docker_exec( - "rabbitmq", - "rabbitmqctl", - "purge_queue", - MAIN_QUEUE_NAME, - check=False, - timeout=90, - ) - self.docker_exec( - "rabbitmq", - "rabbitmqctl", - "purge_queue", - DEAD_LETTER_QUEUE_NAME, - check=False, - timeout=90, - ) - self.docker_exec( - "rabbitmq", - "rabbitmqctl", - "purge_queue", - LEGACY_MAIN_QUEUE_NAME, - check=False, - timeout=90, - ) - - -@pytest.fixture(scope="session") -def compose_env() -> ComposeTestEnv: - _require_docker_tools() - os.environ.setdefault("PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN", "1") - os.environ.setdefault("PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS", "10") - os.environ.setdefault("PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS", "50") - os.environ.setdefault("PIXIVUTIL2_NETWORK_RETRY", "0") - os.environ.setdefault("PIXIVUTIL2_NETWORK_RETRY_WAIT", "1") - env = ComposeTestEnv() - env.compose("down", "--volumes", check=False, timeout=180) - env.compose("up", "--build", "-d", timeout=600) - env.wait_http_ready() - yield env - env.compose("down", "--volumes", check=False, timeout=180) - - -@pytest.fixture -def clean_env(compose_env: ComposeTestEnv) -> ComposeTestEnv: - compose_env.clear_state() - compose_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=30) - compose_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=30) - yield compose_env - compose_env.clear_state() diff --git a/integration_tests/test_dlq_replay.py b/integration_tests/test_dlq_replay.py deleted file mode 100644 index 5def066..0000000 --- a/integration_tests/test_dlq_replay.py +++ /dev/null @@ -1,153 +0,0 @@ -import pytest - -from PixivServer.config.celery import DEAD_LETTER_QUEUE_NAME, MAIN_QUEUE_NAME - - -def _attempt_history(state: dict) -> list[int]: - history = state.get("attempt_history") - if not isinstance(history, list): - return [] - return [int(item) for item in history] - - -@pytest.mark.pixiv_api -def test_dlq_resume_replays_native_celery_message(clean_env): - """ - Replaying a single native Celery DLQ message should republish it back to the main queue and remove it from DLQ. - - This validates the baseline operator workflow for manual recovery using the dev endpoint and DLQ resume endpoint. - If this fails, administrators cannot reliably recover failed tasks from the dead letter queue. - """ - task = clean_env.api_json("POST", "/api/dev/artwork/424242") - task_id = task["task_id"] - - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 1, timeout=60) - messages = clean_env.api_json("GET", "/api/queue/dead-letter/") - assert len(messages) == 1 - message = messages[0] - assert message["dead_letter_id"] == task_id - assert message["task_name"] == "dev_download_artworks_by_id" - assert message["payload"]["artwork_id"] == 424242 - - clean_env.docker_exec( - "pixivutil-worker", - "sh", - "-lc", - "touch /tmp/pixivutil-dev-dlq-success.flag", - ) - - resumed = clean_env.api_json("POST", f"/api/queue/dead-letter/{task_id}/resume") - assert resumed["requeued"] is True - assert resumed["task_name"] == "dev_download_artworks_by_id" - - state = clean_env.wait_for_dev_task_terminal_state(task_id, "succeeded", timeout=90) - assert _attempt_history(state) == [1, 2, 1, 2] - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=60) - assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] - - -@pytest.mark.pixiv_api -def test_dlq_resume_resets_retry_counter_for_replayed_native_celery_message(clean_env): - """ - Replaying a single native Celery message should reset retry state so the task starts a fresh retry lifecycle. - - DLQ replay is an operator recovery action, so the resumed task is expected to behave like a new enqueue rather than - inheriting an exhausted retry budget from the failed message. If this fails, replayed transient failures may skip - retry-attempt 1 and fail immediately or earlier than expected. - """ - artwork_id = 333333 - task = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_id}") - task_id = task["task_id"] - - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 1, timeout=60) - before_state = clean_env.wait_for_dev_task_terminal_state(task_id, "failed", timeout=60) - assert before_state["artwork_id"] == artwork_id - assert _attempt_history(before_state) == [1, 2] - - clean_env.docker_exec( - "pixivutil-worker", - "sh", - "-lc", - "touch /tmp/pixivutil-dev-dlq-success.flag", - ) - - resumed = clean_env.api_json("POST", f"/api/queue/dead-letter/{task_id}/resume") - assert resumed["requeued"] is True - assert resumed["task_name"] == "dev_download_artworks_by_id" - - after_state = clean_env.wait_for_dev_task_terminal_state(task_id, "succeeded", timeout=90) - assert _attempt_history(after_state) == [1, 2, 1, 2] - clean_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=90) - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=90) - - -@pytest.mark.pixiv_api -def test_dlq_resume_all_replays_multiple_native_celery_messages(clean_env): - """ - Bulk DLQ replay should requeue all recognized native Celery messages and empty the dead letter queue. - - This covers the "resume all" operational path used to recover multiple failed tasks at once. If this fails, - operators may believe tasks were recovered while some messages remain stranded in DLQ or are not reprocessed. - """ - task_a = clean_env.api_json("POST", "/api/dev/artwork/111111") - task_b = clean_env.api_json("POST", "/api/dev/artwork/222222") - - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 2, timeout=90) - messages = clean_env.api_json("GET", "/api/queue/dead-letter/") - assert len(messages) == 2 - - ids = {message["dead_letter_id"] for message in messages} - assert ids == {task_a["task_id"], task_b["task_id"]} - assert {message["payload"]["artwork_id"] for message in messages} == {111111, 222222} - - clean_env.docker_exec( - "pixivutil-worker", - "sh", - "-lc", - "touch /tmp/pixivutil-dev-dlq-success.flag", - ) - - resumed = clean_env.api_json("POST", "/api/queue/dead-letter/resume") - assert resumed["requeued"] == 2 - - clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "succeeded", timeout=90) - clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "succeeded", timeout=90) - clean_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=90) - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=90) - assert clean_env.api_json("GET", "/api/queue/dead-letter/") == [] - - -@pytest.mark.pixiv_api -def test_dlq_resume_all_resets_retry_counter_for_replayed_native_celery_messages(clean_env): - """ - Bulk replay should reset retry state for every replayed native Celery message, not only the single-item path. - - The single-message and bulk replay endpoints should have consistent replay semantics. If this fails, bulk recovery - can silently requeue tasks with stale retry metadata, causing immediate re-failure and making DLQ recovery brittle. - """ - artwork_ids = (444444, 555555) - task_a = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_ids[0]}") - task_b = clean_env.api_json("POST", f"/api/dev/artwork/{artwork_ids[1]}") - - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 2, timeout=90) - before_state_a = clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "failed", timeout=90) - before_state_b = clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "failed", timeout=90) - assert _attempt_history(before_state_a) == [1, 2] - assert _attempt_history(before_state_b) == [1, 2] - - clean_env.docker_exec( - "pixivutil-worker", - "sh", - "-lc", - "touch /tmp/pixivutil-dev-dlq-success.flag", - ) - - resumed = clean_env.api_json("POST", "/api/queue/dead-letter/resume") - assert resumed["requeued"] == 2 - - after_state_a = clean_env.wait_for_dev_task_terminal_state(task_a["task_id"], "succeeded", timeout=90) - after_state_b = clean_env.wait_for_dev_task_terminal_state(task_b["task_id"], "succeeded", timeout=90) - assert _attempt_history(after_state_a) == [1, 2, 1, 2] - assert _attempt_history(after_state_b) == [1, 2, 1, 2] - clean_env.wait_for_queue_count(MAIN_QUEUE_NAME, 0, timeout=90) - clean_env.wait_for_queue_count(DEAD_LETTER_QUEUE_NAME, 0, timeout=90) diff --git a/integration_tests/test_queue_priority.py b/integration_tests/test_queue_priority.py deleted file mode 100644 index 4b8ffc0..0000000 --- a/integration_tests/test_queue_priority.py +++ /dev/null @@ -1,131 +0,0 @@ -import random - -import pytest - -from PixivServer.config.celery import ( - LEGACY_MAIN_QUEUE_NAME, - MAIN_QUEUE_NAME, - QUEUE_MAX_PRIORITY, -) - - -@pytest.mark.pixiv_api -def test_high_priority_tasks_are_consecutive_and_preceded_by_at_most_one_low(clean_env): - """ - A later-published pair of high-priority tasks should jump ahead of queued low-priority tasks. - - In live-worker mode (concurrency=1), one low-priority task may already be running or reserved. We accept that, - but the next queued work should prioritize highs. This test proves the operational guarantee that high-priority - tasks are delayed by at most one low task and are not interleaved with queued lows. - """ - sleep_ms = 1200 - low_labels = [f"L{i}" for i in range(1, 6)] - high_labels = ["H1", "H2"] - - for label in low_labels: - clean_env.api_json("POST", f"/api/dev/priority/{label}?priority=1&sleep_ms={sleep_ms}") - for label in high_labels: - clean_env.api_json("POST", f"/api/dev/priority/{label}?priority=3&sleep_ms={sleep_ms}") - - state = clean_env.wait_for_priority_probe_started_count(len(low_labels) + len(high_labels), timeout=120) - started = state.get("started") - assert isinstance(started, list) - - h1_index = started.index("H1") - h2_index = started.index("H2") - assert h2_index == h1_index + 1 - assert h1_index <= 1, f"Expected at most one low before highs, got order={started}" - - -@pytest.mark.pixiv_api -def test_three_tier_priority_orders_h_then_n_then_l_with_at_most_one_leading_low(clean_env): - """ - In live-worker mode, at most one low task may execute before queued priorities take effect. - - After that, queued work should drain by priority order: high -> normal -> low. We randomize submission order - (deterministically) to avoid accidentally testing only a sorted enqueue path. - """ - sleep_ms = 1000 - rng = random.Random(20260224) - - labels = [f"L{i}" for i in range(1, 11)] - labels += [f"N{i}" for i in range(1, 6)] - labels += [f"H{i}" for i in range(1, 6)] - rng.shuffle(labels) - - # Preserve the live-worker edge case we want to allow: one low can start before later priorities arrive. - first_low_index = next(i for i, label in enumerate(labels) if label.startswith("L")) - labels[0], labels[first_low_index] = labels[first_low_index], labels[0] - - for label in labels: - priority = 1 if label.startswith("L") else 2 if label.startswith("N") else 3 - clean_env.api_json("POST", f"/api/dev/priority/{label}?priority={priority}&sleep_ms={sleep_ms}") - - state = clean_env.wait_for_priority_probe_started_count(len(labels), timeout=180) - started = state.get("started") - assert isinstance(started, list) - assert len(started) == len(labels) - - leading_low_count = 1 if started and started[0].startswith("L") else 0 - body = started[leading_low_count:] - - h_block = [label for label in body if label.startswith("H")] - n_block = [label for label in body if label.startswith("N")] - l_block = [label for label in body if label.startswith("L")] - - expected = h_block + n_block + l_block - assert body == expected, f"Expected H* then N* then L* after optional leading L, got order={started}" - - assert len(h_block) == 5, f"Expected 5 high tasks, got {len(h_block)} in order={started}" - assert len(n_block) == 5, f"Expected 5 normal tasks, got {len(n_block)} in order={started}" - assert len(l_block) == 10 - leading_low_count, ( - f"Expected remaining low tasks to be {10 - leading_low_count}, got {len(l_block)} in order={started}" - ) - - -@pytest.mark.pixiv_api -def test_priority_values_above_queue_max_are_not_a_higher_tier_than_max(clean_env): - """ - RabbitMQ x-max-priority should clamp >max values so they behave like max priority, not a new tier. - """ - over_limit = QUEUE_MAX_PRIORITY + 1 - clean_env.api_json("POST", "/api/dev/priority/L1?priority=1&sleep_ms=1800") - clean_env.api_json("POST", "/api/dev/priority/H3?priority=3&sleep_ms=200") - clean_env.api_json("POST", f"/api/dev/priority/HOver?priority={over_limit}&sleep_ms=200") - clean_env.api_json("POST", "/api/dev/priority/N1?priority=2&sleep_ms=200") - - state = clean_env.wait_for_priority_probe_started_count(4, timeout=90) - started = state.get("started") - assert isinstance(started, list) - assert started[0] == "L1" - assert started[1:] == ["H3", "HOver", "N1"], f"Expected over-limit priority to be clamped to max-priority tier, got {started}" - - -@pytest.mark.pixiv_api -def test_worker_startup_deletes_legacy_main_queue(clean_env): - """ - Hard-cutover behavior: worker init should purge/delete the legacy main queue if it exists. - """ - clean_env.seed_legacy_main_queue_message() - clean_env.wait_for_queue_count(LEGACY_MAIN_QUEUE_NAME, 1, timeout=30) - - clean_env.restart_worker() - clean_env.wait_for_queue_absent(LEGACY_MAIN_QUEUE_NAME, timeout=60) - - -@pytest.mark.pixiv_api -def test_worker_restart_keeps_v1_queues_usable_when_they_already_exist(clean_env): - """ - Restarting the worker with pre-existing v1 queues should remain healthy and continue processing tasks. - """ - clean_env.api_json("POST", "/api/dev/priority/BeforeRestart?priority=2&sleep_ms=200") - clean_env.wait_for_priority_probe_started_count(1, timeout=60) - - clean_env.restart_worker() - - clean_env.api_json("POST", "/api/dev/priority/AfterRestart?priority=2&sleep_ms=200") - state = clean_env.wait_for_priority_probe_started_count(2, timeout=90) - started = state.get("started") - assert isinstance(started, list) - assert started[:2] == ["BeforeRestart", "AfterRestart"] - assert MAIN_QUEUE_NAME in clean_env.rabbitmq_queue_counts() diff --git a/pyproject.toml b/pyproject.toml index 0cb29a9..e7eeb89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ pixivutil2 = [ [tool.pyright] pythonVersion = "3.12" -include = ["PixivServer", "PixivServerCommon", "PixivUtilClient", "tests", "integration_tests"] +include = ["PixivServer", "PixivServerCommon", "PixivUtilClient", "tests"] exclude = [ "PixivUtil2", "**/node_modules", From fada1bb18fcbef527e2fb32838a0f1dcca779f8e Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sun, 8 Mar 2026 15:27:48 -0700 Subject: [PATCH 29/32] remove integration test stuff --- PixivServer/service/pixiv.py | 6 ---- PixivServer/worker/common.py | 19 +++++++++++ PixivServer/worker/download.py | 61 ++++++++++++---------------------- PixivServer/worker/metadata.py | 61 ++++++++++++---------------------- docker-compose.yml | 8 ----- 5 files changed, 63 insertions(+), 92 deletions(-) create mode 100644 PixivServer/worker/common.py diff --git a/PixivServer/service/pixiv.py b/PixivServer/service/pixiv.py index bba9e5d..5ec0820 100644 --- a/PixivServer/service/pixiv.py +++ b/PixivServer/service/pixiv.py @@ -105,12 +105,6 @@ def load_environment_variables(self): if pixivutil_config.cookie: __config__.cookie = pixivutil_config.cookie - pixiv_retry = os.getenv("PIXIVUTIL2_NETWORK_RETRY") - if pixiv_retry is not None: - __config__.retry = int(pixiv_retry) - pixiv_retry_wait = os.getenv("PIXIVUTIL2_NETWORK_RETRY_WAIT") - if pixiv_retry_wait is not None: - __config__.retryWait = int(pixiv_retry_wait) return diff --git a/PixivServer/worker/common.py b/PixivServer/worker/common.py new file mode 100644 index 0000000..00fb8ab --- /dev/null +++ b/PixivServer/worker/common.py @@ -0,0 +1,19 @@ +import random +import time +from urllib.error import URLError + +from PixivServer.service.pixiv import PixivException + +NETWORK_MAX_RETRIES = 3 +NETWORK_RETRY_COUNTDOWN = 60 + + +def job_sleep(): + time.sleep(random.uniform(1, 5)) + return 0 + + +def is_network_exception(exc: BaseException) -> bool: + if isinstance(exc, PixivException): + return exc.errorCode in (PixivException.DOWNLOAD_FAILED_NETWORK, PixivException.SERVER_ERROR) + return isinstance(exc, (ConnectionError, TimeoutError, URLError)) diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py index e551b56..a2664bd 100644 --- a/PixivServer/worker/download.py +++ b/PixivServer/worker/download.py @@ -1,9 +1,5 @@ import logging -import os -import random -import time import traceback -from urllib.error import URLError from celery import shared_task @@ -16,30 +12,17 @@ DownloadArtworksByTagsRequest, as_celery_task, ) -from PixivServer.service.pixiv import PixivException +from PixivServer.worker.common import ( + NETWORK_MAX_RETRIES, + NETWORK_RETRY_COUNTDOWN, + is_network_exception, + job_sleep, +) logger = logging.getLogger(__name__) -_NETWORK_MAX_RETRIES = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES", "3")) -_NETWORK_RETRY_COUNTDOWN = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN", "60")) -_JOB_SLEEP_MIN_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS", "1000")) -_JOB_SLEEP_MAX_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS", "5000")) - - -def __job_sleep(): - lower_ms = min(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) - upper_ms = max(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) - time.sleep(random.uniform(lower_ms, upper_ms) / 1000) - return 0 - - -def _is_network_exception(exc: BaseException) -> bool: - if isinstance(exc, PixivException): - return exc.errorCode in (PixivException.DOWNLOAD_FAILED_NETWORK, PixivException.SERVER_ERROR) - return isinstance(exc, (ConnectionError, TimeoutError, URLError)) - -@shared_task(bind=True, name="download_artworks_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="download_artworks_by_id", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def download_artworks_by_id(self, request_dict: dict): try: request = DownloadArtworkByIdRequest(**request_dict) @@ -49,14 +32,14 @@ def download_artworks_by_id(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artworks_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() -@shared_task(bind=True, name="download_artworks_by_member_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="download_artworks_by_member_id", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def download_artworks_by_member_id(self, request_dict: dict): try: request = DownloadArtworksByMemberIdRequest(**request_dict) @@ -66,14 +49,14 @@ def download_artworks_by_member_id(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artworks_by_member_id worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() -@shared_task(bind=True, name="download_artworks_by_tag", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="download_artworks_by_tag", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def download_artworks_by_tag(self, request_dict: dict): try: request = DownloadArtworksByTagsRequest(**request_dict) @@ -83,14 +66,14 @@ def download_artworks_by_tag(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artworks_by_tag worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() -@shared_task(bind=True, name="delete_artwork_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="delete_artwork_by_id", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def delete_artwork_by_id(self, request_dict: dict): try: request = DeleteArtworkByIdRequest(**request_dict) @@ -100,11 +83,11 @@ def delete_artwork_by_id(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in delete_artwork_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() download_artworks_by_id_task = as_celery_task(download_artworks_by_id) diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py index 7b95523..2604b6e 100644 --- a/PixivServer/worker/metadata.py +++ b/PixivServer/worker/metadata.py @@ -1,9 +1,5 @@ import logging -import os -import random -import time import traceback -from urllib.error import URLError from celery import shared_task @@ -16,30 +12,17 @@ DownloadTagMetadataByIdRequest, as_celery_task, ) -from PixivServer.service.pixiv import PixivException +from PixivServer.worker.common import ( + NETWORK_MAX_RETRIES, + NETWORK_RETRY_COUNTDOWN, + is_network_exception, + job_sleep, +) logger = logging.getLogger(__name__) -_NETWORK_MAX_RETRIES = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES", "3")) -_NETWORK_RETRY_COUNTDOWN = int(os.getenv("PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN", "60")) -_JOB_SLEEP_MIN_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS", "1000")) -_JOB_SLEEP_MAX_MS = int(os.getenv("PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS", "5000")) - - -def __job_sleep(): - lower_ms = min(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) - upper_ms = max(_JOB_SLEEP_MIN_MS, _JOB_SLEEP_MAX_MS) - time.sleep(random.uniform(lower_ms, upper_ms) / 1000) - return 0 - - -def _is_network_exception(exc: BaseException) -> bool: - if isinstance(exc, PixivException): - return exc.errorCode in (PixivException.DOWNLOAD_FAILED_NETWORK, PixivException.SERVER_ERROR) - return isinstance(exc, (ConnectionError, TimeoutError, URLError)) - -@shared_task(bind=True, name="download_member_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="download_member_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def download_member_metadata_by_id(self, request_dict: dict): try: request = DownloadMemberMetadataByIdRequest(**request_dict) @@ -52,14 +35,14 @@ def download_member_metadata_by_id(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in download_member_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() -@shared_task(bind=True, name="download_artwork_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="download_artwork_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def download_artwork_metadata_by_id(self, request_dict: dict): try: request = DownloadArtworkMetadataByIdRequest(**request_dict) @@ -72,14 +55,14 @@ def download_artwork_metadata_by_id(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in download_artwork_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() -@shared_task(bind=True, name="download_series_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="download_series_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def download_series_metadata_by_id(self, request_dict: dict): try: request = DownloadSeriesMetadataByIdRequest(**request_dict) @@ -92,14 +75,14 @@ def download_series_metadata_by_id(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in download_series_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() -@shared_task(bind=True, name="download_tag_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=_NETWORK_MAX_RETRIES) +@shared_task(bind=True, name="download_tag_metadata_by_id", queue=MAIN_QUEUE_NAME, max_retries=NETWORK_MAX_RETRIES) def download_tag_metadata_by_id(self, request_dict: dict): try: request = DownloadTagMetadataByIdRequest(**request_dict) @@ -112,11 +95,11 @@ def download_tag_metadata_by_id(self, request_dict: dict): except Exception as e: # noqa: BLE001 logger.error(f"Error in download_tag_metadata_by_id worker: {str(e)}") logger.error(traceback.format_exc()) - if _is_network_exception(e): - raise self.retry(exc=e, countdown=_NETWORK_RETRY_COUNTDOWN) + if is_network_exception(e): + raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) return False finally: - __job_sleep() + job_sleep() download_member_metadata_by_id_task = as_celery_task(download_member_metadata_by_id) diff --git a/docker-compose.yml b/docker-compose.yml index 99f2b82..df14b52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,12 +35,6 @@ services: - PIXIVUTIL_COOKIE=$PIXIVUTIL_COOKIE - PIXIVUTIL_SERVER_ENV=development - RABBITMQ_BROKER_URL=amqp://guest:guest@rabbitmq:5672 - - PIXIVUTIL2_NETWORK_RETRY=${PIXIVUTIL2_NETWORK_RETRY:-3} - - PIXIVUTIL2_NETWORK_RETRY_WAIT=${PIXIVUTIL2_NETWORK_RETRY_WAIT:-5} - - PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES=${PIXIVUTIL_WORKER_NETWORK_MAX_RETRIES:-3} - - PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN=${PIXIVUTIL_WORKER_NETWORK_RETRY_COUNTDOWN:-60} - - PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS=${PIXIVUTIL_WORKER_JOB_SLEEP_MIN_MS:-1000} - - PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS=${PIXIVUTIL_WORKER_JOB_SLEEP_MAX_MS:-5000} volumes: - pixivutil-data:/workdir/.pixivUtil2 - pixivutil-downloads:/workdir/downloads @@ -60,8 +54,6 @@ services: - PIXIVUTIL_SERVER_API_KEY=$PIXIVUTIL_SERVER_API_KEY - PIXIVUTIL_SERVER_ENV=development - RABBITMQ_BROKER_URL=amqp://guest:guest@rabbitmq:5672 - - PIXIVUTIL2_NETWORK_RETRY=${PIXIVUTIL2_NETWORK_RETRY:-3} - - PIXIVUTIL2_NETWORK_RETRY_WAIT=${PIXIVUTIL2_NETWORK_RETRY_WAIT:-5} volumes: - pixivutil-data:/workdir/.pixivUtil2 - pixivutil-downloads:/workdir/downloads From f9b09f128844b4b18033031a40acf161d50cb5a5 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sun, 8 Mar 2026 15:31:54 -0700 Subject: [PATCH 30/32] also clean up exchange --- PixivServer/worker/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/PixivServer/worker/__init__.py b/PixivServer/worker/__init__.py index 56b04da..7f149c8 100644 --- a/PixivServer/worker/__init__.py +++ b/PixivServer/worker/__init__.py @@ -2,12 +2,13 @@ from celery import Celery from celery.signals import setup_logging, worker_init, worker_shutdown -from kombu import Queue +from kombu import Exchange, Queue import PixivServer import PixivServer.service import PixivServer.service.pixiv from PixivServer.config.celery import ( + LEGACY_MAIN_EXCHANGE_NAME, LEGACY_MAIN_QUEUE_NAME, dead_letter_queue, main_queue, @@ -24,6 +25,7 @@ def on_worker_init(sender, **kwargs): with sender.app.connection() as conn: _cleanup_legacy_queue(conn, LEGACY_MAIN_QUEUE_NAME) + _cleanup_legacy_exchange(conn, LEGACY_MAIN_EXCHANGE_NAME) main_queue.bind(conn).declare() dead_letter_queue.bind(conn).declare() PixivServer.service.pixiv.service.open() @@ -56,6 +58,16 @@ def _cleanup_legacy_queue(conn, queue_name: str) -> None: logger.warning(f"Failed to delete legacy queue {queue_name}: {exc}") +def _cleanup_legacy_exchange(conn, exchange_name: str) -> None: + exchange = Exchange(exchange_name, type='direct', durable=True).bind(conn) + try: + exchange.delete() + logger.warning(f"Deleted legacy exchange during v1 broker cutover: {exchange_name}") + except Exception as exc: # noqa: BLE001 + if "NOT_FOUND" not in str(exc): + logger.warning(f"Failed to delete legacy exchange {exchange_name}: {exc}") + + # @celery.on_after_configure.connect # def setup_periodic_tasks(sender, **kwargs): # sender.add_periodic_task(worker_config.subscription_time_seconds, run_artist_subscription_job.s(), name='Artist subscription job') From be7606f22b68fb0f4f15b1511a67512116420956 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sun, 8 Mar 2026 15:52:47 -0700 Subject: [PATCH 31/32] add todo --- PixivServer/worker/download.py | 6 ++++++ PixivServer/worker/metadata.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/PixivServer/worker/download.py b/PixivServer/worker/download.py index a2664bd..454061f 100644 --- a/PixivServer/worker/download.py +++ b/PixivServer/worker/download.py @@ -34,6 +34,9 @@ def download_artworks_by_id(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: non-network errors return False (acked as SUCCESS) to avoid DLQ routing. + # Use per-task acks_on_failure_or_timeout=True + Reject(requeue=False) for network + # exhaustion so non-network errors can raise normally and show as FAILURE. return False finally: job_sleep() @@ -51,6 +54,7 @@ def download_artworks_by_member_id(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: see download_artworks_by_id for non-network error handling fix. return False finally: job_sleep() @@ -68,6 +72,7 @@ def download_artworks_by_tag(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: see download_artworks_by_id for non-network error handling fix. return False finally: job_sleep() @@ -85,6 +90,7 @@ def delete_artwork_by_id(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: see download_artworks_by_id for non-network error handling fix. return False finally: job_sleep() diff --git a/PixivServer/worker/metadata.py b/PixivServer/worker/metadata.py index 2604b6e..88d7607 100644 --- a/PixivServer/worker/metadata.py +++ b/PixivServer/worker/metadata.py @@ -37,6 +37,9 @@ def download_member_metadata_by_id(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: non-network errors return False (acked as SUCCESS) to avoid DLQ routing. + # Use per-task acks_on_failure_or_timeout=True + Reject(requeue=False) for network + # exhaustion so non-network errors can raise normally and show as FAILURE. return False finally: job_sleep() @@ -57,6 +60,7 @@ def download_artwork_metadata_by_id(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: see download_member_metadata_by_id for non-network error handling fix. return False finally: job_sleep() @@ -77,6 +81,7 @@ def download_series_metadata_by_id(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: see download_member_metadata_by_id for non-network error handling fix. return False finally: job_sleep() @@ -97,6 +102,7 @@ def download_tag_metadata_by_id(self, request_dict: dict): logger.error(traceback.format_exc()) if is_network_exception(e): raise self.retry(exc=e, countdown=NETWORK_RETRY_COUNTDOWN) + # TODO: see download_member_metadata_by_id for non-network error handling fix. return False finally: job_sleep() From 42b7efc7ae8facffc5979f7cf5678caddf08154f Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Tue, 10 Mar 2026 21:07:37 -0700 Subject: [PATCH 32/32] add metrics warning logs --- PixivServer/service/metrics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PixivServer/service/metrics.py b/PixivServer/service/metrics.py index b79ffbe..6a001c3 100644 --- a/PixivServer/service/metrics.py +++ b/PixivServer/service/metrics.py @@ -110,8 +110,9 @@ def _rabbitmq_queue_message_count(queue_name: str) -> int | None: with urllib.request.urlopen(req, timeout=5) as resp: data = json.loads(resp.read()) return int(data.get("messages", 0)) - except (urllib.error.URLError, OSError, ValueError): - return None # Management API unavailable + except (urllib.error.URLError, OSError, ValueError) as exc: + logger.warning("RabbitMQ management API unreachable for queue %s: %s", queue_name, exc) + return None def _collect_queue_depth() -> None: