chore: merge main to release

dev/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260604T1454
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260605T2314
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive

dev/build/TARGET_BASE

@@ -1 +1 @@
-20260604T1454
+20260605T2314

dev/build/gunicorn.conf.py

@@ -7,6 +7,10 @@
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
 
 # Bind all ipv4 interfaces (nginx uses loopback, but k8s health checks don't)
 _BIND_PORT = os.environ.get("DATATRACKER_GUNICORN_BIND_PORT", "8000")
@@ -167,15 +171,11 @@ def post_fork(server, worker):
         trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter))
 
         # Instrumentations
-        if "all" in enabled_telemetry or "django" in enabled_telemetry:
-            from opentelemetry.instrumentation.django import DjangoInstrumentor
+        if "all" in enabled_telemetry or "django" in enabled_telemetry: 
             DjangoInstrumentor().instrument()
-        if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry:
-            from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+        if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: 
             Psycopg2Instrumentor().instrument()
-        if "all" in enabled_telemetry or "redis" in enabled_telemetry:
-            from opentelemetry.instrumentation.redis import RedisInstrumentor
-            RedisInstrumentor().instrument()
-        if "all" in enabled_telemetry or "requests" in enabled_telemetry:
-            from opentelemetry.instrumentation.requests import RequestsInstrumentor
+        if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: 
+            PymemcacheInstrumentor().instrument()
+        if "all" in enabled_telemetry or "requests" in enabled_telemetry: 
             RequestsInstrumentor().instrument()

docker-compose.yml

@@ -90,12 +90,6 @@ services:
             - .:/workspace
             - app-assets:/assets
 
-    redis:
-        image: redis:8
-        command: ['redis-server', '--save', '10', '1', '--loglevel', 'warning']
-        volumes:
-            - redis:/data
-
     replicator:
         build:
             context: .
@@ -175,4 +169,3 @@ volumes:
     app-assets:
     minio-data:
     blobdb-data:
-    redis:

docker/base.Dockerfile

@@ -59,10 +59,12 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	libxtst6 \
 	libmagic-dev \
 	libmariadb-dev \
+	libmemcached-tools \
 	libyang2-tools \
 	locales \
 	make \
 	mariadb-client \
+	memcached \
 	nano \
 	netcat-traditional \
 	nodejs \

ietf/ietfauth/utils.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2013-2022, All Rights Reserved
+# Copyright The IETF Trust 2013-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -163,6 +163,7 @@ def has_role(user, role_names, *args, **kwargs):
                 | Q(name="atlarge", group__acronym="irsg")
             ),
             "RSAB Member": Q(name="member", group__acronym="rsab"),
+            "LLC Staff": Q(name="member", group__acronym="llc-staff"),
             "Robot": Q(name="robot", group__acronym="secretariat"),
         }
 

ietf/settings.py

@@ -1363,93 +1363,138 @@ def skip_unreadable_post(record):
     TEMPLATES[0]['OPTIONS']['context_processors'] += DEV_TEMPLATE_CONTEXT_PROCESSORS
 
 if "CACHES" not in locals():
-    # Would like to refuse to start when in prod mode, but this currently blocks
-    # the collectstatics call in the release GHA. We should probably arrange for that
-    # to have its own caches config, but in the meantime just let this fall back to
-    # the dev cache configuration.
-    #
-    # if SERVER_MODE == "production":
-    #     raise RuntimeError("Must set CACHES in settings_local for production mode")
-    CACHES = {
-        "default": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            # "BACKEND": "django_redis.cache.RedisCache",
-            # "LOCATION": "redis://redis:6379/0",
-            # "OPTIONS": {
-            #     "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
-            # },
-            # "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
-            "VERSION": __version__,
-            "KEY_PREFIX": "ietf:dt",
-        },
-        "agenda": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            # "BACKEND": "django_redis.cache.RedisCache",
-            # "LOCATION": "redis://redis:6379/0",
-            # "OPTIONS": {
-            #     "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
-            # },
-            # No release-specific VERSION setting.
-            "KEY_PREFIX": "ietf:dt:agenda",
-            # Key function is default except with sha384-encoded key
-            "KEY_FUNCTION": lambda key, key_prefix, version: (
-                f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
-            ),
-        },
-        "proceedings": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            # "BACKEND": "django_redis.cache.RedisCache",
-            # "LOCATION": "redis://redis:6379/0",
-            # "OPTIONS": {
-            #     "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
-            # },
-            # No release-specific VERSION setting.
-            "KEY_PREFIX": "ietf:dt:proceedings",
-            # Key function is default except with sha384-encoded key
-            "KEY_FUNCTION": lambda key, key_prefix, version: (
-                f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
-            ),
-        },
-        "sessions": {
-            "BACKEND": "django_redis.cache.RedisCache",
-            "LOCATION": "redis://redis:6379/0",
-            "OPTIONS": {
-                "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient",
+    if SERVER_MODE == "production":
+        MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1")
+        MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211")
+        CACHES = {
+            "default": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                "VERSION": __version__,
+                "KEY_PREFIX": "ietf:dt",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
             },
-        },
-        "htmlized": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-            "LOCATION": "/var/cache/datatracker/htmlized",
-            "OPTIONS": {
-                "MAX_ENTRIES": 1000,
+            "agenda": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:agenda",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
             },
-        },
-        "pdfized": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-            "LOCATION": "/var/cache/datatracker/pdfized",
-            "OPTIONS": {
-                "MAX_ENTRIES": 1000,
+            "proceedings": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:proceedings",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
             },
-        },
-        "slowpages": {
-            "BACKEND": "django.core.cache.backends.dummy.DummyCache",
-            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-            "LOCATION": "/var/cache/datatracker/",
-            "OPTIONS": {
-                "MAX_ENTRIES": 5000,
+            "sessions": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt",
             },
-        },
-        "celery-results": {
-            "BACKEND": "django_redis.cache.RedisCache",
-            "LOCATION": "redis://redis:6379/0",
-            "OPTIONS": {
-                "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            "htmlized": {
+                "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+                "LOCATION": "/a/cache/datatracker/htmlized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 100000,  # 100,000
+                },
             },
-            "KEY_PREFIX": "ietf:celery",
-        },
-    }
+            "pdfized": {
+                "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+                "LOCATION": "/a/cache/datatracker/pdfized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 100000,  # 100,000
+                },
+            },
+            "slowpages": {
+                "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+                "LOCATION": "/a/cache/datatracker/slowpages",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 5000,
+                },
+            },
+            "celery-results": {
+                "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                "KEY_PREFIX": "ietf:celery",
+            },
+        }
+    else:
+        CACHES = {
+            "default": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'ietf.utils.cache.LenientMemcacheCache',
+                #'LOCATION': '127.0.0.1:11211',
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "VERSION": __version__,
+                "KEY_PREFIX": "ietf:dt",
+            },
+            "agenda": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                # "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                # "LOCATION": "127.0.0.1:11211",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:agenda",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
+            "proceedings": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                # "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                # "LOCATION": "127.0.0.1:11211",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:proceedings",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
+            "sessions": {
+                "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+            },
+            "htmlized": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "LOCATION": "/var/cache/datatracker/htmlized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 1000,
+                },
+            },
+            "pdfized": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "LOCATION": "/var/cache/datatracker/pdfized",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 1000,
+                },
+            },
+            "slowpages": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+                "LOCATION": "/var/cache/datatracker/",
+                "OPTIONS": {
+                    "MAX_ENTRIES": 5000,
+                },
+            },
+            "celery-results": {
+                "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+                "LOCATION": "app:11211",
+                "KEY_PREFIX": "ietf:celery",
+            },
+        }
 
 PUBLISH_IPR_STATES = ['posted', 'removed', 'removed_objfalse']
 
@@ -1464,6 +1509,7 @@ def skip_unreadable_post(record):
     loaders = TEMPLATES[0]['OPTIONS']['loaders']
     loaders = tuple(l for e in loaders for l in (e[1] if isinstance(e, tuple) and "cached.Loader" in e[0] else (e,)))
     TEMPLATES[0]['OPTIONS']['loaders'] = loaders
+    SESSION_ENGINE = "django.contrib.sessions.backends.db"
 
     if 'SECRET_KEY' not in locals():
         SECRET_KEY = 'PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHka'

ietf/settings_test.py

@@ -69,9 +69,6 @@ def tempdir_with_cleanup(**kwargs):
 PHOTOS_DIR = os.path.join(MEDIA_ROOT, PHOTOS_DIRNAME)
 os.mkdir(PHOTOS_DIR)
 
-# Use database-backed sessions for tests
-SESSION_ENGINE = "django.contrib.sessions.backends.db"
-
 # Undo any developer-dependent middleware when running the tests
 MIDDLEWARE = [ c for c in MIDDLEWARE if not c in DEV_MIDDLEWARE ] # pyflakes:ignore