diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 4aca4c0e1eb..72bdb7a7ae1 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260604T1454 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260605T2314 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index b64be08c8c7..7ca47fd1979 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260604T1454 +20260605T2314 diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index a1c572a0965..03e81eac5e2 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -7,6 +7,10 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor +from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor # Bind all ipv4 interfaces (nginx uses loopback, but k8s health checks don't) _BIND_PORT = os.environ.get("DATATRACKER_GUNICORN_BIND_PORT", "8000") @@ -167,15 +171,11 @@ def post_fork(server, worker): trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) # Instrumentations - if "all" in enabled_telemetry or "django" in enabled_telemetry: - from opentelemetry.instrumentation.django import DjangoInstrumentor + if "all" in enabled_telemetry or "django" in enabled_telemetry: DjangoInstrumentor().instrument() - if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: - from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor + if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: Psycopg2Instrumentor().instrument() - if "all" in enabled_telemetry or "redis" in enabled_telemetry: - from opentelemetry.instrumentation.redis import RedisInstrumentor - RedisInstrumentor().instrument() - if "all" in enabled_telemetry or "requests" in enabled_telemetry: - from opentelemetry.instrumentation.requests import RequestsInstrumentor + if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: + PymemcacheInstrumentor().instrument() + if "all" in enabled_telemetry or "requests" in enabled_telemetry: RequestsInstrumentor().instrument() diff --git a/docker-compose.yml b/docker-compose.yml index f171fb261bd..073d04b896a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,12 +90,6 @@ services: - .:/workspace - app-assets:/assets - redis: - image: redis:8 - command: ['redis-server', '--save', '10', '1', '--loglevel', 'warning'] - volumes: - - redis:/data - replicator: build: context: . @@ -175,4 +169,3 @@ volumes: app-assets: minio-data: blobdb-data: - redis: diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 7bf4263b382..25016360494 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -59,10 +59,12 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends libxtst6 \ libmagic-dev \ libmariadb-dev \ + libmemcached-tools \ libyang2-tools \ locales \ make \ mariadb-client \ + memcached \ nano \ netcat-traditional \ nodejs \ diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 0df667fbd2f..30d51cddd09 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -163,6 +163,7 @@ def has_role(user, role_names, *args, **kwargs): | Q(name="atlarge", group__acronym="irsg") ), "RSAB Member": Q(name="member", group__acronym="rsab"), + "LLC Staff": Q(name="member", group__acronym="llc-staff"), "Robot": Q(name="robot", group__acronym="secretariat"), } diff --git a/ietf/settings.py b/ietf/settings.py index 6d9dc00e0e0..95f2ffefd76 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1363,93 +1363,138 @@ def skip_unreadable_post(record): TEMPLATES[0]['OPTIONS']['context_processors'] += DEV_TEMPLATE_CONTEXT_PROCESSORS if "CACHES" not in locals(): - # Would like to refuse to start when in prod mode, but this currently blocks - # the collectstatics call in the release GHA. We should probably arrange for that - # to have its own caches config, but in the meantime just let this fall back to - # the dev cache configuration. - # - # if SERVER_MODE == "production": - # raise RuntimeError("Must set CACHES in settings_local for production mode") - CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django_redis.cache.RedisCache", - # "LOCATION": "redis://redis:6379/0", - # "OPTIONS": { - # "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient", - # }, - # "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "VERSION": __version__, - "KEY_PREFIX": "ietf:dt", - }, - "agenda": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django_redis.cache.RedisCache", - # "LOCATION": "redis://redis:6379/0", - # "OPTIONS": { - # "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient", - # }, - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt:agenda", - # Key function is default except with sha384-encoded key - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "proceedings": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - # "BACKEND": "django_redis.cache.RedisCache", - # "LOCATION": "redis://redis:6379/0", - # "OPTIONS": { - # "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient", - # }, - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt:proceedings", - # Key function is default except with sha384-encoded key - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "sessions": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://redis:6379/0", - "OPTIONS": { - "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingRedisClient", + if SERVER_MODE == "production": + MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") + MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") + CACHES = { + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), }, - }, - "htmlized": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - "LOCATION": "/var/cache/datatracker/htmlized", - "OPTIONS": { - "MAX_ENTRIES": 1000, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), }, - }, - "pdfized": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - "LOCATION": "/var/cache/datatracker/pdfized", - "OPTIONS": { - "MAX_ENTRIES": 1000, + "proceedings": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), }, - }, - "slowpages": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - "LOCATION": "/var/cache/datatracker/", - "OPTIONS": { - "MAX_ENTRIES": 5000, + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt", }, - }, - "celery-results": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://redis:6379/0", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", + "htmlized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, }, - "KEY_PREFIX": "ietf:celery", - }, - } + "pdfized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/slowpages", + "OPTIONS": { + "MAX_ENTRIES": 5000, + }, + }, + "celery-results": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "KEY_PREFIX": "ietf:celery", + }, + } + else: + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + #'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', + #'LOCATION': '127.0.0.1:11211', + #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + }, + "agenda": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "sessions": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + }, + "htmlized": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + "LOCATION": "/var/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 1000, + }, + }, + "pdfized": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + "LOCATION": "/var/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 1000, + }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + "LOCATION": "/var/cache/datatracker/", + "OPTIONS": { + "MAX_ENTRIES": 5000, + }, + }, + "celery-results": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "LOCATION": "app:11211", + "KEY_PREFIX": "ietf:celery", + }, + } PUBLISH_IPR_STATES = ['posted', 'removed', 'removed_objfalse'] @@ -1464,6 +1509,7 @@ def skip_unreadable_post(record): loaders = TEMPLATES[0]['OPTIONS']['loaders'] loaders = tuple(l for e in loaders for l in (e[1] if isinstance(e, tuple) and "cached.Loader" in e[0] else (e,))) TEMPLATES[0]['OPTIONS']['loaders'] = loaders + SESSION_ENGINE = "django.contrib.sessions.backends.db" if 'SECRET_KEY' not in locals(): SECRET_KEY = 'PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHka' diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 883a582a28a..e7ebc13eb28 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -69,9 +69,6 @@ def tempdir_with_cleanup(**kwargs): PHOTOS_DIR = os.path.join(MEDIA_ROOT, PHOTOS_DIRNAME) os.mkdir(PHOTOS_DIR) -# Use database-backed sessions for tests -SESSION_ENGINE = "django.contrib.sessions.backends.db" - # Undo any developer-dependent middleware when running the tests MIDDLEWARE = [ c for c in MIDDLEWARE if not c in DEV_MIDDLEWARE ] # pyflakes:ignore diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index dc5b5d6ae88..d8e87417026 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,8 +1,10 @@ # Copyright The IETF Trust 2016-2026, All Rights Reserved import calendar -import json +import csv import datetime +import io +import json from django.http import Http404 from pyquery import PyQuery @@ -18,10 +20,12 @@ import ietf.stats.views -from ietf.group.factories import RoleFactory -from ietf.person.factories import PersonFactory +from ietf.doc.factories import NewRevisionDocEventFactory +from ietf.group.factories import GroupFactory, RoleFactory +from ietf.person.factories import EmailFactory, PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory +from ietf.submit.factories import SubmissionFactory from ietf.utils.timezone import date_today @@ -183,3 +187,139 @@ def test_review_stats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) + + +class AnnualReportInputsTests(TestCase): + def setUp(self): + super().setUp() + llc_staff = GroupFactory(acronym="llc-staff", type_id="team") + self.member = PersonFactory() + RoleFactory(group=llc_staff, name_id="member", person=self.member) + self.non_member = PersonFactory() + + def _member_login(self): + self.client.login( + username=self.member.user.username, + password=f"{self.member.user.username}+password", + ) + + def test_access_unauthenticated(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertIn("/accounts/login", r["Location"]) + + def test_access_non_member_forbidden(self): + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + self.client.login( + username=self.non_member.user.username, + password=f"{self.non_member.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + + def test_access_member_allowed(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2024"}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + + def test_default_year(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], datetime.date.today().year - 1) + + def test_year_param_redirects_to_year_url(self): + self._member_login() + url = urlreverse(ietf.stats.views.annual_report_inputs) + r = self.client.get(url, {"year": "2022"}) + self.assertRedirects( + r, + urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": "2022"}), + ) + + def test_summary_counts(self): + self._member_login() + year = 2021 + # author1 has a matching Person record; author2 does not + EmailFactory(address="author1@example.com") + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 6, 1), + submitter_email="submitter@example.com", + ) + sub.authors = [ + {"name": "Author One", "email": "author1@example.com", "affiliation": "", "country": "", "errors": []}, + {"name": "Author Two", "email": "author2@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 6, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + # Same draft, second revision — should count once + extra = NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 1, tzinfo=datetime.timezone.utc), + doc__type_id="draft", + ) + NewRevisionDocEventFactory( + time=datetime.datetime(year, 9, 15, tzinfo=datetime.timezone.utc), + doc=extra.doc, + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["year"], year) + self.assertEqual(r.context["author_count"], 2) + self.assertEqual(r.context["author_person_count"], 1) + self.assertEqual(r.context["author_noperson_count"], 1) + self.assertEqual(r.context["submitter_count"], 1) + self.assertEqual(r.context["submitter_person_count"], 0) + self.assertEqual(r.context["submitter_noperson_count"], 1) + self.assertEqual(r.context["draft_count"], 3) + + def test_download_authors_csv(self): + self._member_login() + year = 2020 + sub = SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + ) + sub.authors = [ + {"name": "Author", "email": "csvauthor@example.com", "affiliation": "", "country": "", "errors": []}, + ] + sub.save() + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "authors"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"authors-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvauthor@example.com", rows[0]) + + def test_download_submitters_csv(self): + self._member_login() + year = 2020 + SubmissionFactory( + state_id="posted", + submission_date=datetime.date(year, 4, 1), + submitter_email="csvsubmitter@example.com", + ) + + url = urlreverse(ietf.stats.views.annual_report_inputs, kwargs={"year": str(year)}) + r = self.client.get(url, {"download": "submitters"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "text/csv") + self.assertIn(f"submitters-{year}.csv", r["Content-Disposition"]) + rows = list(csv.reader(io.StringIO(r.content.decode()))) + self.assertEqual(len(rows), 1) + self.assertIn("csvsubmitter@example.com", rows[0]) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 01b8758c840..3bb107813a6 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,4 +15,5 @@ url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), + url(r"^annual_report_inputs/(?:(?P\d{4})/)?$", views.annual_report_inputs), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index d61c9cab64d..fe2fa82f554 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,8 +1,9 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- import calendar +import csv import datetime import itertools import json @@ -12,7 +13,7 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.cache import cache -from django.http import HttpResponseRedirect +from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render, get_object_or_404 from django.urls import reverse as urlreverse from django.db.models import Count @@ -28,7 +29,7 @@ from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName from ietf.meeting.models import Registration, Meeting -from ietf.ietfauth.utils import has_role +from ietf.ietfauth.utils import has_role, role_required from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO from ietf.meeting.helpers import get_current_ietf_meeting_num @@ -957,3 +958,46 @@ def time_key_fn(t): "possible_states": possible_states, "selected_state": selected_state, }) + + +@role_required("LLC Staff") +def annual_report_inputs(request, year=None): + if year is None and "year" in request.GET: + return HttpResponseRedirect( + urlreverse("ietf.stats.views.annual_report_inputs", kwargs={"year": request.GET["year"]}) + ) + year = int(year) if year else datetime.date.today().year - 1 + + from ietf.doc.models import NewRevisionDocEvent + from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people + + download = request.GET.get("download") + if download in ("authors", "submitters"): + addresses = authors_by_year(year) if download == "authors" else submitters_by_year(year) + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="{download}-{year}.csv"' + writer = csv.writer(response) + writer.writerow(sorted(addresses)) + return response + + authors = authors_by_year(year) + submitters = submitters_by_year(year) + author_persons, author_nopersons = unique_people(authors) + submitter_persons, submitter_nopersons = unique_people(submitters) + + draft_count = len(set( + NewRevisionDocEvent.objects.filter( + doc__type_id="draft", time__year=year + ).values_list("doc__name", flat=True) + )) + + return render(request, "stats/annual_report_inputs.html", { + "year": year, + "author_count": len(authors), + "submitter_count": len(submitters), + "author_person_count": author_persons.count(), + "author_noperson_count": len(author_nopersons), + "submitter_person_count": submitter_persons.count(), + "submitter_noperson_count": len(submitter_nopersons), + "draft_count": draft_count, + }) diff --git a/ietf/submit/factories.py b/ietf/submit/factories.py index 1076434b822..69888a055dd 100644 --- a/ietf/submit/factories.py +++ b/ietf/submit/factories.py @@ -21,6 +21,9 @@ class Meta: class SubmissionFactory(factory.django.DjangoModelFactory): state_id = 'uploaded' + submitter_name = factory.Faker("name") + submitter_email = factory.Faker("email") + submitter = factory.LazyAttribute(lambda o: f"{o.submitter_name} <{o.submitter_email}>") @factory.lazy_attribute_sequence def name(self, n): @@ -32,3 +35,4 @@ def auth_key(self): class Meta: model = Submission + exclude = ("submitter_name", "submitter_email") diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 43ca025e28b..b6b744ec21e 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} {% origin %} {% load ietf_filters managed_groups wg_menu active_groups_menu group_filters cache meetings_filters %} @@ -453,6 +453,14 @@ {% endif %} + {% if user|has_role:"LLC Staff" %} +
  • + + Annual report inputs + +
    • + {% endif %}
  • diff --git a/ietf/templates/stats/annual_report_inputs.html b/ietf/templates/stats/annual_report_inputs.html new file mode 100644 index 00000000000..15add7ece3c --- /dev/null +++ b/ietf/templates/stats/annual_report_inputs.html @@ -0,0 +1,55 @@ +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

    {% block title %}Annual Report Inputs for {{ year }}{% endblock %}

    +
    +
    + + + +
    +
    +

    Summary

    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Email addressesUnique persons foundAddresses with no person recordApproximate unique people
    Authors{{ author_count }}{{ author_person_count }}{{ author_noperson_count }}{{ author_person_count|add:author_noperson_count }}
    Submitters{{ submitter_count }}{{ submitter_person_count }}{{ submitter_noperson_count }}{{ submitter_person_count|add:submitter_noperson_count }}
    +

    Drafts submitted in {{ year }}: {{ draft_count }}

    +

    Downloads

    + +{% endblock %} diff --git a/ietf/utils/cache.py b/ietf/utils/cache.py index 60dd2c3ae34..0baa56da2da 100644 --- a/ietf/utils/cache.py +++ b/ietf/utils/cache.py @@ -1,67 +1,20 @@ -# Copyright The IETF Trust 2026, All Rights Reserved -from typing import Optional, Union, Any +# Copyright The IETF Trust 2023, All Rights Reserved +# -*- coding: utf-8 -*- -from django.core.cache import BaseCache from django.core.cache.backends.base import DEFAULT_TIMEOUT -from django_redis.client import DefaultClient, SentinelClient -from redis import Redis -from redis.typing import KeyT, EncodableT +from django.core.cache.backends.memcached import PyMemcacheCache +from pymemcache.exceptions import MemcacheServerError -from ietf.utils.log import log +from .log import log -class EncodedValueTooBig(ValueError): - def __init__(self, *args, value_len): - super().__init__(*args) - self.value_len = value_len - - -class SizeLimitingRedisClient(DefaultClient): - """Redis DefaultClient that refuses to cache large objects - - Size applies to the literal cached value, which is _after_ serialization and - compression. - - Set size limit with MAX_ENCODED_VALUE_LEN in the OPTIONS dict. Defaults to - 1 MB. - """ - def __init__(self, server, params: dict[str, Any], backend: BaseCache) -> None: - super().__init__(server, params, backend) - self.max_encoded_value_len = self._options.get("MAX_ENCODED_VALUE_LEN", 1 << 20) - - def encode(self, value: EncodableT) -> Union[bytes, int]: - encoded = super().encode(value) - if isinstance(encoded, bytes) and len(encoded) > self.max_encoded_value_len: - raise EncodedValueTooBig(value_len=len(encoded)) - return encoded - - def set( - self, - key: KeyT, - value: EncodableT, - timeout: Optional[float] = DEFAULT_TIMEOUT, - version: Optional[int] = None, - client: Optional[Redis] = None, - nx: bool = False, - xx: bool = False, - ) -> bool: +class LenientMemcacheCache(PyMemcacheCache): + """PyMemcacheCache backend that tolerates failed inserts due to object size""" + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): try: - return super().set(key, value, timeout, version, client, nx, xx) - except EncodedValueTooBig as err: - log( - f"Refused to cache large object for {key!r} " - f"({err.value_len} > {self.max_encoded_value_len} bytes)" - ) - return False - - -class SizeLimitingSentinelClient(SizeLimitingRedisClient, SentinelClient): - """Redis SentinelClient that refuses to cache large objects - - Size applies to the literal cached value, which is _after_ serialization and - compression. - - Set size limit with MAX_ENCODED_VALUE_LEN in the OPTIONS dict. Defaults to - 1 MB. - """ - pass + super().set(key, value, timeout, version) + except MemcacheServerError as err: + if "object too large for cache" in str(err): + log(f"Memcache failed to cache large object for {key}") + else: + raise diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py new file mode 100755 index 00000000000..9a969a52178 --- /dev/null +++ b/ietf/utils/reports.py @@ -0,0 +1,49 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved + +from typing import List, Set, Tuple +from django.db.models import QuerySet + +from email.utils import parseaddr + +from ietf.person.models import Person +from ietf.submit.models import Submission + + +def authors_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" + addresses = set() + for submission in Submission.objects.filter( + submission_date__year=year, state="posted" + ): + addresses.update([a["email"] for a in submission.authors]) + return addresses + + +def submitters_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" + return set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year, state="posted" + ).values_list("submitter", flat=True) + ] + ) + + +def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: + """Identify Person records matching email addresses and email addresses with no Person record. + + Given a list of email addresses, return + ( + a list of unique Person records with a matching email address, + a list of unique email addresses with no matching Person record + ) + The sum of the lengths of these lists is a best-approximation for how + many unique people the list of addresses belong to. + """ + persons = Person.objects.filter(email__address__in=addresses).distinct() + known_email = set(persons.values_list("email__address", flat=True)) + return (persons, set(addresses) - set(known_email)) + + diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index 6a8f4529a80..4e1ee27895b 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -171,6 +171,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: "acronym": rfc.group.acronym, "name": rfc.group.name, "full": f"{rfc.group.acronym} - {rfc.group.name}", + "type": rfc.group.type.slug, } if ( rfc.group.parent is not None diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py new file mode 100755 index 00000000000..83daa15cc14 --- /dev/null +++ b/ietf/utils/tests_reports.py @@ -0,0 +1,189 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime + +import debug # pyflakes:ignore + +from ietf.doc.factories import IndividualDraftFactory +from ietf.person.factories import EmailFactory +from ietf.person.models import Person, Email +from ietf.submit.factories import SubmissionFactory +from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people +from ietf.utils.test_utils import TestCase + + +class ReportTests(TestCase): + def setUp(self): + super().setUp() + + # Build 5 drafts submitted across two years, with 6 unique authors, + # one of which has more than one email address (author0@example.com and + # author0@example.net). The drafts are submitted by two of the six authors, + # again using multiple addresses for author0, and two identities that are not authors. + # Then build a draft where the submission's submitter info doesn't contain an email + # address (we have those in the production database) to make sure that submitter isn't + # counted. + + self.make_draft_submission( + year=2020, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 1", "author1@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=3, + day=3, + submitter_name="NotanAuthor 0", + submitter_email="notanauthor0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Note alternate email + ("Author 3", "author3@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=12, + day=31, + submitter_name="Author 3", + submitter_email="author3@example.net", + author_nameaddrs=[("Author 3", "author3@example.net")], + ) + + self.make_draft_submission( + year=2021, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.com", # Note alternate email + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Again, alternate email + ("Author 4", "author4@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="NoatanAuthor 2", + submitter_email="notanauthor2@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 3", "author3@example.net"), + ("Author 5", "author5@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="Trouble Maker", + submitter_email="", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 2", "author2@example.net"), + ], + ) + + def make_draft_submission( + self, year, month, day, submitter_name, submitter_email, author_nameaddrs + ): + + authors = [] + for name, addr in author_nameaddrs: + person = Person.objects.filter(name=name).first() + if not person: + person = EmailFactory(person__name=name, address=addr).person + elif not Email.objects.filter(address=addr).exists(): + EmailFactory(person=person, address=addr) + authors.append(person) + + submission = SubmissionFactory( + submission_date=datetime.date(year, month, day), + submitter_name=submitter_name, + submitter_email=submitter_email, + state_id="posted", + ) + submission.authors = [ + { + "name": f"{name}", + "email": f"{addr}", + "affiliation": "", + "country": "", + "errors": [], + } + for name, addr in author_nameaddrs + ] + + submission.save() + IndividualDraftFactory(submission=submission, authors=authors) + + def test_authors_by_year(self): + authors2020 = authors_by_year(2020) + self.assertEqual( + authors2020, + set( + [ + "author0@example.net", + "author0@example.com", + "author1@example.net", + "author3@example.net", + ] + ), + ) + authors2021 = authors_by_year(2021) + self.assertEqual( + authors2021, + set( + [ + "author0@example.net", + "author0@example.com", + "author2@example.net", + "author3@example.net", + "author4@example.net", + "author5@example.net", + ] + ), + ) + + def test_submitters_by_year(self): + sub2020 = submitters_by_year(2020) + self.assertEqual( + sub2020, + set( + [ + "author0@example.net", + "author3@example.net", + "notanauthor0@example.net", + ] + ), + ) + sub2021 = submitters_by_year(2021) + self.assertEqual( + sub2021, set(["author0@example.com", "notanauthor2@example.net"]) + ) + + def test_unique_people(self): + persons, addrs = unique_people( + [ + "notanauthor0@example.com", + "author0@example.net", + "author0@example.com", + "author1@example.net", + "notanauthor0@example.com", + ] + ) + self.assertEqual(addrs, set(["notanauthor0@example.com"])) + self.assertEqual( + set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) + ) + self.assertEqual(len(persons) + len(addrs), 3) diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index b1e278a9149..769cb03517e 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -12,5 +12,6 @@ resources: - beat.yaml - celery.yaml - datatracker.yaml + - memcached.yaml - rabbitmq.yaml - replicator.yaml diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml new file mode 100644 index 00000000000..68b732d7459 --- /dev/null +++ b/k8s/memcached.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: memcached + template: + metadata: + labels: + app: memcached + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # Memcached + # ----------------------------------------------------- + - image: "memcached:1.6-alpine" + imagePullPolicy: IfNotPresent + args: ["-m", "1024"] + name: memcached + ports: + - name: memcached + containerPort: 11211 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # memcached image sets up uid/gid 11211 + runAsUser: 11211 + runAsGroup: 11211 + resources: + requests: + cpu: 100m + memory: 100Mi + # ----------------------------------------------------- + # Memcached Exporter for Prometheus + # ----------------------------------------------------- + - image: "quay.io/prometheus/memcached-exporter:v0.14.3" + imagePullPolicy: IfNotPresent + name: memcached-exporter + ports: + - name: metrics + containerPort: 9150 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 65534 # nobody + runAsGroup: 65534 # nobody + resources: + requests: + cpu: 10m + memory: 20Mi + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached + annotations: + k8s.grafana.com/scrape: "true" # this is not a bool + k8s.grafana.com/metrics.portName: "metrics" +spec: + type: ClusterIP + ports: + - port: 11211 + targetPort: memcached + protocol: TCP + name: memcached + - port: 9150 + targetPort: metrics + protocol: TCP + name: metrics + selector: + app: memcached diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 560648c07f6..20c5252ff03 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -315,29 +315,15 @@ def _multiline_to_list(s): DE_GFM_BINARY = "/usr/local/bin/de-gfm" IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" +# Duplicating production cache from settings.py and using it whether we're in production mode or not +MEMCACHED_HOST = os.environ.get("DT_MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("DT_MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ -# Common config for redis caches -REDIS_SENTINEL_SERVICE = os.environ.get("DATATRACKER_REDIS_SENTINEL_HOST") -REDIS_SENTINEL_PORT = os.environ.get("DATATRACKER_REDIS_SENTINEL_PORT", "26379") -DJANGO_REDIS_CONNECTION_FACTORY = "django_redis.pool.SentinelConnectionFactory" -REDIS_CACHE_CONFIG_COMMON = { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://dt-master/0", - "OPTIONS": { - "CLIENT_CLASS": "ietf.utils.cache.SizeLimitingSentinelClient", - "MAX_ENCODED_VALUE_LEN": int( - os.environ.get("DATATRACKER_REDIS_MAX_ENCODED_VALUE_LEN", 1 << 20) - ), - "SENTINELS": [(REDIS_SENTINEL_SERVICE, REDIS_SENTINEL_PORT)], - "SENTINEL_KWARGS": {}, - "CONNECTION_POOL_CLASS": "redis.sentinel.SentinelConnectionPool", - }, -} - CACHES = { - "default": REDIS_CACHE_CONFIG_COMMON - | { + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", "VERSION": __version__, "KEY_PREFIX": "ietf:dt", # Key function is default except with sha384-encoded key @@ -345,8 +331,9 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, - "agenda": REDIS_CACHE_CONFIG_COMMON - | { + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:agenda", # Key function is default except with sha384-encoded key @@ -354,8 +341,9 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, - "proceedings": REDIS_CACHE_CONFIG_COMMON - | { + "proceedings": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt:proceedings", # Key function is default except with sha384-encoded key @@ -363,8 +351,9 @@ def _multiline_to_list(s): f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, - "sessions": REDIS_CACHE_CONFIG_COMMON - | { + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", # No release-specific VERSION setting. "KEY_PREFIX": "ietf:dt", }, @@ -389,8 +378,9 @@ def _multiline_to_list(s): "MAX_ENTRIES": 5000, }, }, - "celery-results": REDIS_CACHE_CONFIG_COMMON - | { + "celery-results": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", "KEY_PREFIX": "ietf:celery", }, } diff --git a/requirements.txt b/requirements.txt index bf7d1d26e0b..31e8ea69d1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,6 @@ django-debug-toolbar>=6.0.0 django-filter>=24.3 django-markup>=1.10 # Limited use - need to reconcile against direct use of markdown django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return -django-redis>=6.0.0 django-simple-history>=3.10.1 django-storages>=1.14.6 django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below @@ -59,8 +58,8 @@ oic>=1.7.0 # Used only by tests opentelemetry-sdk>=1.38.0 opentelemetry-instrumentation-django>=0.59b0 opentelemetry-instrumentation-psycopg2>=0.59b0 +opentelemetry-instrumentation-pymemcache>=0.59b0 opentelemetry-instrumentation-requests>=0.59b0 -opentelemetry-instrumentation-redis>=0.63b1 opentelemetry-exporter-otlp-proto-http>=1.38.0 pillow>=11.3.0 psycopg2>=2.9.10 @@ -73,9 +72,11 @@ python-dateutil>=2.9.0 types-python-dateutil>=2.9.0 python-json-logger>=3.3.0 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures +pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields types-pytz==2025.2.0.20251108 # match pytz version +typesense>=2.0.0 requests>=2.32.4 types-requests>=2.32.4 requests-mock>=1.12.1 @@ -85,7 +86,6 @@ selenium>=4.34.2 tblib>=3.1.0 # So that the django test runner provides tracebacks tlds>=2022042700 # Used to teach bleach about which TLDs currently exist tqdm>=4.67.1 -typesense>=2.0.0 unidecode>=1.4.0 urllib3>=2.5.0 weasyprint>=66.0