Run Yente with workers and indexer on the side. Document how to increase tunables for Yente.

Author: apognu

.env.example

@@ -2,7 +2,7 @@
 # REQUIRED SETTINGS
 #
 
-# If equal to "development", applies configurations on the backend and frontend containers that 
+# If equal to "development", applies configurations on the backend and frontend containers that
 # are not suitable for production, related to CORS, secure cookies and https.
 # Any other value works for a production environment, and will be used by sentry to mark the environment
 # an error occurred in.
@@ -32,10 +32,10 @@ MARBLE_API_URL_CLIENT="http://localhost:8080"
 # RSA private key, in PEM format, used for for signing authentication tokens. MUST be changed for production.
 # We recommend using AUTHENTICATION_JWT_SIGNING_KEY_FILE to point to a private key on disk because multi-line
 # environment variables are not handled very gracefully.
-# 
+#
 # If using AUTHENTICATION_JWT_SIGNING_KEY, line breaks should be replaced with '\n':
 # e.g. AUTHENTICATION_JWT_SIGNING_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBA...
-# 
+#
 # To generate a private key, use `openssl genrsa -out /path/to/private/key.pem 4096`.
 # If left empty, a key will be regenerated on every app restart, which may cause unexpected logouts.
 AUTHENTICATION_JWT_SIGNING_KEY=
@@ -81,7 +81,7 @@ INGESTION_BUCKET_URL="file://./tempFiles/data-ingestion-bucket?create_dir=true"
 CASE_MANAGER_BUCKET_URL="file://./tempFiles/case-manager-bucket?create_dir=true"
 
 # Configure your Firebase project to allow username and password authentication.
-# To retrieve this value, go into the settings of your Firebase project, and register 
+# To retrieve this value, go into the settings of your Firebase project, and register
 # a new app from the 'General' tab.
 FIREBASE_API_KEY=
 
@@ -105,7 +105,7 @@ CREATE_ORG_ADMIN_EMAIL=
 
 # Configure the connection details to your Metabase instance.
 # - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding'
-# - The dashboard ID is the ID of the dashboard you want to embed in Marble. You will find it when you publish a dashboard as 
+# - The dashboard ID is the ID of the dashboard you want to embed in Marble. You will find it when you publish a dashboard as
 #   a static embedding on Metabase.
 # - The site url can be on a metabase domain, or can use your self-hosted Metabase instance.
 METABASE_SITE_URL=
@@ -115,7 +115,7 @@ METABASE_GLOBAL_DASHBOARD_ID=
 # Set up connection details to Convoy to enable webhooks sending.
 # You can get your project ID and API key from your project settings page in Convoy's dashboard, in the "Secrets" section.
 # NB: CONVOY_API_URL should be {scheme}://{host}:{port}/api - forgetting the /api will result in unexpected errors.
-CONVOY_API_URL= 
+CONVOY_API_URL=
 CONVOY_API_KEY=
 CONVOY_PROJECT_ID=
 
@@ -124,6 +124,8 @@ CONVOY_PROJECT_ID=
 # when using docker-compose.
 YENTE_ELASTICSEARCH_URL=http://marble-es:9200
 YENTE_PORT=8000
+YENTE_API_WORKERS=4
+YENTE_QUERY_CONCURRENCY=50
 ES_PORT=9200
 
 # Configure access to the Open Sanctions API to use sanction checks
@@ -141,6 +143,6 @@ SENTRY_DSN=
 
 # If you need to support federated authentication through Firebase, you will need to configure the following settings:
 # To retrieve those values, go into the settings of your Firebase project, and register a new app from the 'General' tab.
-# The two pieces of information below will be provided to you after that. 
+# The two pieces of information below will be provided to you after that.
 # FIREBASE_APP_ID=
 # FIREBASE_AUTH_DOMAIN=

docker-compose.yaml

@@ -160,14 +160,34 @@ services:
       - marble-es:/usr/share/elasticsearch/data
   yente:
     container_name: marble-yente
-    image: ghcr.io/opensanctions/yente:4.2.1
+    image: ghcr.io/opensanctions/yente:4.2.3
     depends_on:
       - elasticsearch
+    command:
+      [
+        "uvicorn",
+        "--host=0.0.0.0",
+        "--workers=${YENTE_API_WORKERS:-4}",
+        "yente.asgi:app",
+      ]
     ports:
       - ${YENTE_PORT:-8000}:${YENTE_PORT:-8000}
     environment:
       YENTE_INDEX_TYPE: elasticsearch
-      YENTE_INDEX_URL: "${YENTE_ELASTICSEARCH_URL}"
+      YENTE_INDEX_URL: ${YENTE_ELASTICSEARCH_HOST:-http://marble-es:9200}
+      YENTE_QUERY_CONCURRENCY: ${YENTE_QUERY_CONCURRENCY:-50}
+      YENTE_UPDATE_TOKEN: ""
+      YENTE_AUTO_REINDEX: false
+  yente-indexer:
+    container_name: marble-yente-indexer
+    image: ghcr.io/opensanctions/yente:4.2.3
+    depends_on:
+      - elasticsearch
+    command: ["yente", "reindex"]
+    environment:
+      YENTE_INDEX_TYPE: elasticsearch
+      YENTE_INDEX_URL: ${YENTE_ELASTICSEARCH_HOST:-http://marble-es:9200}
+      YENTE_QUERY_CONCURRENCY: "50"
       YENTE_UPDATE_TOKEN: ""
 
 volumes:

installation/production_run.md

@@ -131,6 +131,10 @@ Options:
 
 > ⚠️ **Note**: While development docker-compose includes Elasticsearch, use a production-grade service for deployment.
 
+It is recommended to deploy the Yente API with multiple workers (see the `docker-compose.yaml` to see how) and disable the indexing process on those with `YENTE_AUTO_REINDEX=false`) so it doesn't impact production workloads and is not duplicated across workers.
+
+Once automatic background indexing is disabled, you will need to run it (with `yente reindex`) separately through a different container or a scheduled task (cron or systemd timer).
+
 ## Deployment Architecture
 
 ### Components