From 89e7833bf77861feed4adcda3b5de4200c462af7 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Wed, 19 Mar 2025 22:20:50 -0300 Subject: [PATCH 01/28] feat: add basic files for a deployment only with docker compose --- deploy/.staging.env | 36 ++++ deploy/README.md | 11 + deploy/docker-compose.staging.yml | 67 ++++++ deploy/kratos/config.staging.yml | 191 ++++++++++++++++++ deploy/kratos/config.yml | 191 ++++++++++++++++++ deploy/kratos/identity.schema.json | 51 +++++ deploy/kratos/identity_id.jsonnet | 1 + deploy/kratos/nbp_user_mapper.jsonnet | 13 ++ .../subscribe_newsletter_mapper.jsonnet | 18 ++ deploy/kratos/user_mapper.jsonnet | 11 + deploy/kratos/vidis_user_mapper.jsonnet | 40 ++++ deploy/nginx.staging.conf | 25 +++ 12 files changed, 655 insertions(+) create mode 100644 deploy/.staging.env create mode 100644 deploy/README.md create mode 100644 deploy/docker-compose.staging.yml create mode 100644 deploy/kratos/config.staging.yml create mode 100644 deploy/kratos/config.yml create mode 100644 deploy/kratos/identity.schema.json create mode 100644 deploy/kratos/identity_id.jsonnet create mode 100644 deploy/kratos/nbp_user_mapper.jsonnet create mode 100644 deploy/kratos/subscribe_newsletter_mapper.jsonnet create mode 100644 deploy/kratos/user_mapper.jsonnet create mode 100644 deploy/kratos/vidis_user_mapper.jsonnet create mode 100644 deploy/nginx.staging.conf diff --git a/deploy/.staging.env b/deploy/.staging.env new file mode 100644 index 000000000..0a35edd32 --- /dev/null +++ b/deploy/.staging.env @@ -0,0 +1,36 @@ +# This a template. Replace the values with the actual ones in the environment. + +ENVIRONMENT=staging +GOOGLE_SPREADSHEET_API_ACTIVE_DONORS=1qpyC0XzvTcKT6EISywvqESX3A0MwQoFDE8p-Bll4hps +GOOGLE_SPREADSHEET_API_MOTIVATION=142gKytX3rMT25DiIwQvhTLZrjZlojzZ4APIvwRuR4tU +GOOGLE_SPREADSHEET_API_SECRET=my-secret +LOG_LEVEL=ERROR +MAILCHIMP_API_KEY=secret-us5 +METADATA_API_VERSION=1.0.0 +MYSQL_URI=mysql://root:secret@mysql:3306/serlo?timezone=+00:00 +REDIS_URL=redis://redis:6379 +ROCKET_CHAT_API_USER_ID=an-user-id +ROCKET_CHAT_API_AUTH_TOKEN=an-auth-token +ROCKET_CHAT_URL=https://community.serlo.org/ +SERLO_ORG_DATABASE_LAYER_HOST=127.0.0.1:8080 +SERLO_ORG_SECRET=serlo.org-secret +# Set the following value to `empty` to disable caching +CACHE_TYPE=redis + +SERVER_HYDRA_HOST=http://hydra:4445 +SERVER_KRATOS_PUBLIC_HOST=http://kratos:4433 +SERVER_KRATOS_ADMIN_HOST=http://kratos:4434 +SERVER_KRATOS_SECRET=api.serlo.org-kratos-secret +SERVER_KRATOS_DB_URI=postgres://serlo:secret@postgres:5432/kratos?sslmode=disable\&max_conns=20\&max_idle_conns=4 +SERVER_SERLO_CLOUDFLARE_WORKER_SECRET=api.serlo.org-playground-secret +SERVER_SERLO_CACHE_WORKER_SECRET=api.serlo.org-cache-worker-secret +SERVER_SERLO_NOTIFICATION_EMAIL_SERVICE_SECRET=api.serlo.org-notification-email-service-secret +SERVER_SERLO_EDITOR_TESTING_SECRET=api.serlo.org-serlo-editor-testing-secret +SERVER_SWR_QUEUE_DASHBOARD_PASSWORD=secret +SERVER_SWR_QUEUE_DASHBOARD_USERNAME=secret + +ENMESHED_SERVER_HOST="http://enmeshed:8081/" +ENMESHED_SERVER_SECRET="apiKey" +ENMESHED_WEBHOOK_SECRET="webhookKey" + +OPENAI_API_KEY="" diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 000000000..40a24495a --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,11 @@ +# Deployment Instructions with Docker Compose + +## Requirements +- Docker +- Nginx + +## Steps for the Staging deployment + +1. Set up Nginx on the host machine using configuration file `nginx.staging.conf`. +2. Be sure the values at `.staging.env` are correct. +3. Deploy using Docker Compose with file `docker-compose.staging.yml`. diff --git a/deploy/docker-compose.staging.yml b/deploy/docker-compose.staging.yml new file mode 100644 index 000000000..8c393ec33 --- /dev/null +++ b/deploy/docker-compose.staging.yml @@ -0,0 +1,67 @@ +services: + api: + image: ghcr.io/serlo/api.serlo.org/server:staging + pull_policy: always + env_file: + - .staging.env + ports: + - '3001:3001' + # TODO best practice: healthcheck + depends_on: + - mysql + - redis + swr-queue-worker: + image: ghcr.io/serlo/api.serlo.org/swr-queue-worker:staging + pull_policy: always + env_file: + - .staging.env + # TODO: ports needed? + ports: + - '3000:3000' + # TODO best practice: healthcheck + depends_on: + - mysql + - redis + redis: + image: redis:7 + mysql: + image: mysql:8 + environment: + - MYSQL_ROOT_PASSWORD=secret + - MYSQL_DATABASE=serlo + - MYSQL_USER=serlo + - MYSQL_PASSWORD=secret + volumes: + - mysql_data:/var/lib/mysql + kratos: + image: oryd/kratos:v1.3.0 + ports: + - '4433:4433' # public + environment: + - DSN=postgres://serlo:secret@postgres:5432/kratos?sslmode=disable + depends_on: + - postgres + - kratos-migrate + command: serve -c /etc/config/kratos/config.yml --dev --watch-courier + volumes: + - ./kratos:/etc/config/kratos + kratos-migrate: + image: oryd/kratos:v1.3.0 + command: -c /etc/config/kratos/config.yml migrate sql -e --yes + depends_on: + - postgres + volumes: + - ./kratos:/etc/config/kratos + restart: on-failure + postgres: + image: postgres:13 + environment: + - POSTGRES_USER=serlo + - POSTGRES_PASSWORD=secret + - POSTGRES_DB=kratos + volumes: + - postgres_data:/var/lib/postgresql/data + +volumes: + postgres_data: + mysql_data: \ No newline at end of file diff --git a/deploy/kratos/config.staging.yml b/deploy/kratos/config.staging.yml new file mode 100644 index 000000000..045287343 --- /dev/null +++ b/deploy/kratos/config.staging.yml @@ -0,0 +1,191 @@ +dsn: postgres://serlo:secret@postgres:5432/kratos?sslmode=disable&max_conns=20&max_idle_conns=4 + +serve: + public: + base_url: https://kratos.serlo-staging.dev + request_log: + disable_for_health: true + admin: + request_log: + disable_for_health: true + +selfservice: + default_browser_return_url: https://serlo-staging.dev/ + allowed_return_urls: + # TODO: try with wildcard later + - https://fr.serlo-staging.dev/ + - https://hi.serlo-staging.dev/ + - https://de.serlo-staging.dev/ + - https://ta.serlo-staging.dev/ + - https://en.serlo-staging.dev/ + - https://es.serlo-staging.dev/ + methods: + password: + enabled: true + config: + haveibeenpwned_enabled: false + link: + enabled: true + config: + base_url: https://serlo-staging.dev/api/.ory/ + oidc: + enabled: true + config: + base_redirect_uri: https://serlo-staging.dev/api/.ory/ + providers: + - id: nbp + provider: generic + client_id: ${nbp_client_id} + client_secret: ${nbp_client_secret} + issuer_url: https://aai.demo.meinbildungsraum.de/realms/nbp-aai + mapper_url: base64://${nbp_user_mapper} + - id: vidis + provider: generic + client_id: ${vidis_client_id} + client_secret: ${vidis_client_secret} + issuer_url: ${vidis_issuer_url} + mapper_url: base64://${vidis_user_mapper} + + flows: + error: + ui_url: https://serlo-staging.dev/auth/error + + settings: + ui_url: https://serlo-staging.dev/auth/settings + privileged_session_max_age: 15m + + recovery: + enabled: true + use: link + ui_url: https://serlo-staging.dev/auth/recovery + + verification: + enabled: true + use: link + ui_url: https://serlo-staging.dev/auth/verification + + logout: + after: + default_browser_return_url: https://serlo-staging.dev/auth/login + + login: + ui_url: https://serlo-staging.dev/auth/login + lifespan: 10m + after: + password: + hooks: + - hook: require_verified_address + - hook: web_hook + config: + url: https://api.serlo-staging.dev/kratos/updateLastLogin + method: POST + body: base64://${user_id_mapper} + response: + ignore: true + oidc: + default_browser_return_url: https://serlo-staging.dev/auth/login + + registration: + enable_legacy_one_step: true + lifespan: 10m + ui_url: https://serlo-staging.dev/auth/registration + after: + hooks: + - hook: web_hook + config: + url: https://api.serlo-staging.dev/kratos/register + method: POST + body: base64://${user_id_mapper} + auth: + type: api_key + config: + name: x-kratos-key + value: ${kratos_secret} + in: header + - hook: web_hook + config: + url: https://${mailchimp_server}.api.mailchimp.com/3.0/lists/a7bb2bbc4f/members + method: POST + body: base64://${subscribe_newsletter_mapper} + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: ${newsletter_api_key} + oidc: + default_browser_return_url: https://serlo-staging.dev/auth/login + hooks: + - hook: web_hook + config: + url: https://api.serlo-staging.dev/kratos/register + method: POST + body: base64://${user_id_mapper} + auth: + type: api_key + config: + name: x-kratos-key + value: ${kratos_secret} + in: header + - hook: session + - hook: web_hook + config: + url: https://${mailchimp_server}.api.mailchimp.com/3.0/lists/a7bb2bbc4f/members + method: POST + body: base64://${subscribe_newsletter_mapper} + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: ${newsletter_api_key} + +session: + lifespan: 720h + +secrets: + cookie: + - ${cookie_secret} + +identity: + default_schema_id: default + schemas: + - id: default + url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtp://SMTP_Injection:${smtp_password}@smtp.eu.sparkpostmail.com:2525 + from_name: Serlo + from_address: no-reply@mail.serlo.org + templates: + verification: + valid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/verification/valid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/verification/valid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/verification/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/verification/invalid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/verification/invalid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/verification/invalid/email.body.plaintext.gotmpl + recovery: + valid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/recovery/valid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/recovery/valid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/recovery/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/recovery/invalid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/recovery/invalid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/recovery/invalid/email.body.plaintext.gotmpl diff --git a/deploy/kratos/config.yml b/deploy/kratos/config.yml new file mode 100644 index 000000000..aa93adbb6 --- /dev/null +++ b/deploy/kratos/config.yml @@ -0,0 +1,191 @@ +# TODO: remove when config.staging.yml has the correct values + +version: v1.0.0 + +dev: true + +dsn: postgres://serlo:secret@postgres:5432/kratos?sslmode=disable&max_conns=20&max_idle_conns=4 + +serve: + public: + base_url: http://localhost:4433/ + admin: + base_url: http://localhost:4434/ + +selfservice: + default_browser_return_url: http://localhost:3000/ + + methods: + password: + enabled: true + config: + haveibeenpwned_enabled: false + min_password_length: 6 + identifier_similarity_check_enabled: false + link: + enabled: true + oidc: + enabled: true + config: + providers: + - id: nbp + provider: generic + client_id: serlo + client_secret: H8t6WKWtGwFjqNfuAjqxrwCfsdznMAfj + issuer_url: http://nbp:11111/realms/master + mapper_url: file:///etc/config/kratos/user_mapper.jsonnet + - id: vidis + provider: generic + client_id: serlo + client_secret: H8t6WKWtGwFjqNfuAjqxrwCfsdznMAfj + issuer_url: http://vidis:11112/realms/master + mapper_url: file:///etc/config/kratos/user_mapper.jsonnet + + flows: + error: + ui_url: http://localhost:3000/auth/error + logout: + after: + default_browser_return_url: http://localhost:3000/auth/login + login: + ui_url: http://localhost:3000/auth/login + lifespan: 10m + after: + password: + hooks: + - hook: require_verified_address + - hook: web_hook + config: + url: http://host.docker.internal:3001/kratos/updateLastLogin + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + response: + ignore: true + oidc: + default_browser_return_url: http://localhost:3000/auth/login + + registration: + ui_url: http://localhost:3000/auth/registration + after: + hooks: + - hook: web_hook + config: + url: http://host.docker.internal:3001/kratos/register + method: POST + can_interrupt: true + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: api.serlo.org-kratos-secret + in: header + - hook: web_hook + config: + url: https://.api.mailchimp.com/3.0/lists//members + method: POST + body: file:///etc/config/kratos/add_to_newsletter.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PUT_HERE_YOUR_MAILCHIMP_API_KEY + + oidc: + hooks: + - hook: web_hook + config: + url: http://host.docker.internal:3001/kratos/register + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: api.serlo.org-kratos-secret + in: header + - hook: web_hook + config: + url: https://.api.mailchimp.com/3.0/lists//members + method: POST + body: file:///etc/config/kratos/add_to_newsletter.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PUT_HERE_YOUR_MAILCHIMP_API_KEY + - hook: session + default_browser_return_url: http://localhost:3000/auth/login + verification: + enabled: true + use: link + ui_url: http://localhost:3000/auth/verification + recovery: + enabled: true + use: link + ui_url: http://localhost:3000/auth/recovery + + settings: + ui_url: http://localhost:3000/auth/settings +log: + level: debug + format: text + leak_sensitive_values: true + +secrets: + cookie: + - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE + cipher: + - 32-LONG-SECRET-NOT-SECURE-AT-ALL + +ciphers: + algorithm: xchacha20-poly1305 + +hashers: + algorithm: bcrypt + bcrypt: + cost: 8 + +identity: + default_schema_id: default + schemas: + - id: default + url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtps://test:secret@mailslurper:1025/?skip_ssl_verify=true + template_override_path: /etc/config/kratos/email-templates/ + templates: + verification: + valid: + email: + subject: http://host.docker.internal:3000/api/.ory/mail-templates/verification/valid/email.subject.gotmpl + body: + html: http://host.docker.internal:3000/api/.ory/mail-templates/verification/valid/email.body.gotmpl + plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/verification/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://host.docker.internal:3000/api/.ory/mail-templates/verification/invalid/email.subject.gotmpl + body: + html: http://host.docker.internal:3000/api/.ory/mail-templates/verification/invalid/email.body.gotmpl + plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/verification/invalid/email.body.plaintext.gotmpl + recovery: + valid: + email: + subject: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/valid/email.subject.gotmpl + body: + html: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/valid/email.body.gotmpl + plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/invalid/email.subject.gotmpl + body: + html: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/invalid/email.body.gotmpl + plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/invalid/email.body.plaintext.gotmpl diff --git a/deploy/kratos/identity.schema.json b/deploy/kratos/identity.schema.json new file mode 100644 index 000000000..3826a6d2d --- /dev/null +++ b/deploy/kratos/identity.schema.json @@ -0,0 +1,51 @@ +{ + "$id": "https://serlo.org/auth/kratos-identity.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "User", + "type": "object", + "properties": { + "traits": { + "type": "object", + "properties": { + "email": { + "type": "string", + "format": "email", + "ory.sh/kratos": { + "credentials": { "password": { "identifier": true } }, + "verification": { "via": "email" }, + "recovery": { "via": "email" } + } + }, + "username": { + "type": "string", + "ory.sh/kratos": { + "credentials": { "password": { "identifier": true } } + }, + "pattern": "^[\\w\\-]+$", + "maxLength": 32 + }, + "subscribedNewsletter": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "motivation": { + "type": "string" + }, + "profileImage": { + "type": "string" + }, + "language": { + "type": "string" + }, + "interest": { + "type": "string", + "enum": ["parent", "teacher", "pupil", "student", "other", ""] + } + }, + "required": ["email", "username", "interest"], + "additionalProperties": false + } + } +} diff --git a/deploy/kratos/identity_id.jsonnet b/deploy/kratos/identity_id.jsonnet new file mode 100644 index 000000000..f380aac95 --- /dev/null +++ b/deploy/kratos/identity_id.jsonnet @@ -0,0 +1 @@ +function(ctx) { userId: ctx.identity.id } diff --git a/deploy/kratos/nbp_user_mapper.jsonnet b/deploy/kratos/nbp_user_mapper.jsonnet new file mode 100644 index 000000000..cac7eb7e8 --- /dev/null +++ b/deploy/kratos/nbp_user_mapper.jsonnet @@ -0,0 +1,13 @@ +local claims = std.extVar('claims'); + +local enshortenUuid(uuid) = std.split(uuid, '-')[0]; + +{ + identity: { + traits: { + email: enshortenUuid(claims.sub) + '@nbp', + username: enshortenUuid(claims.sub), + interest: '', + }, + }, +} diff --git a/deploy/kratos/subscribe_newsletter_mapper.jsonnet b/deploy/kratos/subscribe_newsletter_mapper.jsonnet new file mode 100644 index 000000000..a75c19130 --- /dev/null +++ b/deploy/kratos/subscribe_newsletter_mapper.jsonnet @@ -0,0 +1,18 @@ +local getInterestsKey = function(interest) + if interest == 'parent' then 'dec9a97288' + else if interest == 'teacher' then '05a5ab768a' + else if interest == 'pupil' then 'bbffc7a064' + else if interest == 'student' then 'ebff3b63f6' + else if interest == 'other' then 'd251aad97e'; + +function(ctx) { + email_address: if 'subscribedNewsletter' in ctx.identity.traits && ctx.identity.traits.subscribedNewsletter == true then + ctx.identity.traits.email + else + error 'User did not subscribe to newsletter. Aborting!', + merge_fields: { + UNAME: ctx.identity.traits.username, + }, + status: 'subscribed', + [if 'interest' in ctx.identity.traits then 'interests' else null]: { [getInterestsKey(ctx.identity.traits.interest)]: true }, +} diff --git a/deploy/kratos/user_mapper.jsonnet b/deploy/kratos/user_mapper.jsonnet new file mode 100644 index 000000000..3e8d7be3e --- /dev/null +++ b/deploy/kratos/user_mapper.jsonnet @@ -0,0 +1,11 @@ +local claims = std.extVar('claims'); + +{ + identity: { + traits: { + [if "email" in claims then "email" else null]: claims.email, + username: claims.preferred_username + "-1232kjl", + interest: "", + }, + }, +} \ No newline at end of file diff --git a/deploy/kratos/vidis_user_mapper.jsonnet b/deploy/kratos/vidis_user_mapper.jsonnet new file mode 100644 index 000000000..194581cad --- /dev/null +++ b/deploy/kratos/vidis_user_mapper.jsonnet @@ -0,0 +1,40 @@ +local claims = std.extVar('claims'); + +local enshortenUuid(uuid) = std.split(uuid, '-')[0]; + +local extractFromClaims = function(fieldName) + if fieldName in claims then claims[fieldName] else null; + +local uuid = extractFromClaims('sub'); + +local buildEmail = function() + local email = extractFromClaims('email'); + + if email != '' && email != null + then email + else enshortenUuid(uuid) + '@fakeemail.vidis'; + +local buildUsername = function() + local preferredUsername = extractFromClaims('preferred_username'); + local truncatedUsername = if std.length(preferredUsername) > 23 + then std.substr(preferredUsername, 0, 23) + else preferredUsername; + + if truncatedUsername != '' && truncatedUsername != null + then truncatedUsername + '-' + enshortenUuid(uuid) + else enshortenUuid(uuid); + +local checkIfIsTeacher = function() + local rawClaims = extractFromClaims('raw_claims'); + + if 'rolle' in rawClaims then rawClaims.rolle == 'LEHR' else false; + +if checkIfIsTeacher() then { + identity: { + traits: { + email: buildEmail(), + username: buildUsername(), + interest: 'other', + }, + }, +} else error 'ERR_BAD_ROLE' diff --git a/deploy/nginx.staging.conf b/deploy/nginx.staging.conf new file mode 100644 index 000000000..04b1f422c --- /dev/null +++ b/deploy/nginx.staging.conf @@ -0,0 +1,25 @@ +server { + listen 80; + server_name serlo-staging.dev; + + location / { + proxy_pass http://localhost:3001; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + server_name kratos.serlo-staging.dev; + + location / { + proxy_pass http://localhost:4433; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} \ No newline at end of file From 094164dc5bf86b85623272c9419efba971fff818 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 20 Mar 2025 10:56:14 -0300 Subject: [PATCH 02/28] use files in the kratos config instead of base64 --- deploy/kratos/config.staging.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/kratos/config.staging.yml b/deploy/kratos/config.staging.yml index 045287343..7cc978b2e 100644 --- a/deploy/kratos/config.staging.yml +++ b/deploy/kratos/config.staging.yml @@ -38,13 +38,13 @@ selfservice: client_id: ${nbp_client_id} client_secret: ${nbp_client_secret} issuer_url: https://aai.demo.meinbildungsraum.de/realms/nbp-aai - mapper_url: base64://${nbp_user_mapper} + mapper_url: file:///etc/config/kratos/user_mapper.jsonnet - id: vidis provider: generic client_id: ${vidis_client_id} client_secret: ${vidis_client_secret} issuer_url: ${vidis_issuer_url} - mapper_url: base64://${vidis_user_mapper} + mapper_url: file:///etc/config/kratos/vidis_user_mapper.jsonnet flows: error: @@ -79,7 +79,7 @@ selfservice: config: url: https://api.serlo-staging.dev/kratos/updateLastLogin method: POST - body: base64://${user_id_mapper} + body: file:///etc/config/kratos/identity_id.jsonnet response: ignore: true oidc: @@ -95,7 +95,7 @@ selfservice: config: url: https://api.serlo-staging.dev/kratos/register method: POST - body: base64://${user_id_mapper} + body: file:///etc/config/kratos/identity_id.jsonnet auth: type: api_key config: @@ -106,7 +106,7 @@ selfservice: config: url: https://${mailchimp_server}.api.mailchimp.com/3.0/lists/a7bb2bbc4f/members method: POST - body: base64://${subscribe_newsletter_mapper} + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet can_interrupt: false response: ignore: true @@ -122,7 +122,7 @@ selfservice: config: url: https://api.serlo-staging.dev/kratos/register method: POST - body: base64://${user_id_mapper} + body: file:///etc/config/kratos/identity_id.jsonnet auth: type: api_key config: @@ -134,7 +134,7 @@ selfservice: config: url: https://${mailchimp_server}.api.mailchimp.com/3.0/lists/a7bb2bbc4f/members method: POST - body: base64://${subscribe_newsletter_mapper} + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet can_interrupt: false response: ignore: true From b0cd800697bffbe39435f428b21797810dcefb1e Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Fri, 21 Mar 2025 07:55:19 -0300 Subject: [PATCH 03/28] refactor(deploy): add placeholders to staging config for kratos --- deploy/README.md | 2 +- deploy/kratos/config.staging.yml | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 40a24495a..64d203aa6 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -7,5 +7,5 @@ ## Steps for the Staging deployment 1. Set up Nginx on the host machine using configuration file `nginx.staging.conf`. -2. Be sure the values at `.staging.env` are correct. +2. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. 3. Deploy using Docker Compose with file `docker-compose.staging.yml`. diff --git a/deploy/kratos/config.staging.yml b/deploy/kratos/config.staging.yml index 7cc978b2e..c0437dfab 100644 --- a/deploy/kratos/config.staging.yml +++ b/deploy/kratos/config.staging.yml @@ -35,15 +35,15 @@ selfservice: providers: - id: nbp provider: generic - client_id: ${nbp_client_id} - client_secret: ${nbp_client_secret} + client_id: PLACEHOLDER + client_secret: PLACEHOLDER issuer_url: https://aai.demo.meinbildungsraum.de/realms/nbp-aai mapper_url: file:///etc/config/kratos/user_mapper.jsonnet - id: vidis provider: generic - client_id: ${vidis_client_id} - client_secret: ${vidis_client_secret} - issuer_url: ${vidis_issuer_url} + client_id: PLACEHOLDER + client_secret: PLACEHOLDER + issuer_url: http://PLACEHOLDER mapper_url: file:///etc/config/kratos/vidis_user_mapper.jsonnet flows: @@ -100,11 +100,11 @@ selfservice: type: api_key config: name: x-kratos-key - value: ${kratos_secret} + value: PLACEHOLDER in: header - hook: web_hook config: - url: https://${mailchimp_server}.api.mailchimp.com/3.0/lists/a7bb2bbc4f/members + url: https://PLACEHOLDER method: POST body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet can_interrupt: false @@ -114,7 +114,7 @@ selfservice: type: basic_auth config: user: serlo - password: ${newsletter_api_key} + password: PLACEHOLDER oidc: default_browser_return_url: https://serlo-staging.dev/auth/login hooks: @@ -127,12 +127,12 @@ selfservice: type: api_key config: name: x-kratos-key - value: ${kratos_secret} + value: PLACEHOLDER in: header - hook: session - hook: web_hook config: - url: https://${mailchimp_server}.api.mailchimp.com/3.0/lists/a7bb2bbc4f/members + url: https://PLACEHOLDER method: POST body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet can_interrupt: false @@ -142,14 +142,14 @@ selfservice: type: basic_auth config: user: serlo - password: ${newsletter_api_key} + password: PLACEHOLDER session: lifespan: 720h secrets: cookie: - - ${cookie_secret} + - PLACEHOLDER identity: default_schema_id: default @@ -159,7 +159,7 @@ identity: courier: smtp: - connection_uri: smtp://SMTP_Injection:${smtp_password}@smtp.eu.sparkpostmail.com:2525 + connection_uri: smtp://PLACEHOLDER from_name: Serlo from_address: no-reply@mail.serlo.org templates: From 4f6cc4e47d2ecf95e65886618f5e17e81115d1af Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:42:57 -0300 Subject: [PATCH 04/28] fix configuration of nginx --- deploy/nginx.staging.conf | 46 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/deploy/nginx.staging.conf b/deploy/nginx.staging.conf index 04b1f422c..b1aad9027 100644 --- a/deploy/nginx.staging.conf +++ b/deploy/nginx.staging.conf @@ -1,25 +1,33 @@ -server { - listen 80; - server_name serlo-staging.dev; +worker_processes auto; - location / { - proxy_pass http://localhost:3001; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } +events { + worker_connections 1024; } -server { - listen 80; - server_name kratos.serlo-staging.dev; +http { + server { + listen 80; + server_name serlo-staging.dev; + + location / { + proxy_pass http://localhost:3001; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + + server { + listen 80; + server_name kratos.serlo-staging.dev; - location / { - proxy_pass http://localhost:4433; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + location / { + proxy_pass http://localhost:4433; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } } } \ No newline at end of file From 8af3836f12fe60705a5a3d8a4f8b58ee85a69d06 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Wed, 26 Mar 2025 14:39:23 -0300 Subject: [PATCH 05/28] doc(deploy): give more instructions --- deploy/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/README.md b/deploy/README.md index 64d203aa6..00f25365b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -7,5 +7,12 @@ ## Steps for the Staging deployment 1. Set up Nginx on the host machine using configuration file `nginx.staging.conf`. +```console +$ cp nginx.staging.conf /etc/nginx/sites-available/default +$ systemctl restart nginx +``` 2. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. 3. Deploy using Docker Compose with file `docker-compose.staging.yml`. +```console +$ docker compose -f docker-compose.staging.yml up -d +``` From 311ad23fd243c96db9409ea18f6e1ff56226fec6 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Wed, 26 Mar 2025 14:40:24 -0300 Subject: [PATCH 06/28] refactor(deploy): use config for nginx as site available --- deploy/nginx.staging.conf | 48 ++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/deploy/nginx.staging.conf b/deploy/nginx.staging.conf index b1aad9027..0b5fe684e 100644 --- a/deploy/nginx.staging.conf +++ b/deploy/nginx.staging.conf @@ -1,33 +1,25 @@ -worker_processes auto; +server { + listen 80; + server_name serlo-staging.dev; -events { - worker_connections 1024; -} - -http { - server { - listen 80; - server_name serlo-staging.dev; - - location / { - proxy_pass http://localhost:3001; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } + location / { + proxy_pass http://localhost:3001; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; } +} - server { - listen 80; - server_name kratos.serlo-staging.dev; +server { + listen 80; + server_name kratos.serlo-staging.dev; - location / { - proxy_pass http://localhost:4433; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } + location / { + proxy_pass http://localhost:4433; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; } -} \ No newline at end of file +} From 5a06d9717332a5895c7a59f09bf0658ced1b4f95 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 27 Mar 2025 20:00:24 -0300 Subject: [PATCH 07/28] chore(deploy-with-docker): improve documentation --- deploy/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 00f25365b..d2bf502e5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -3,16 +3,18 @@ ## Requirements - Docker - Nginx +- Git ## Steps for the Staging deployment -1. Set up Nginx on the host machine using configuration file `nginx.staging.conf`. +1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org` +2. Set up Nginx on the host machine using configuration file `nginx.staging.conf`. ```console -$ cp nginx.staging.conf /etc/nginx/sites-available/default -$ systemctl restart nginx +$ sudo cp nginx.staging.conf /etc/nginx/sites-available/default +$ sudo systemctl restart nginx ``` -2. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. -3. Deploy using Docker Compose with file `docker-compose.staging.yml`. +3. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. +4. Deploy using Docker Compose with file `docker-compose.staging.yml`. ```console $ docker compose -f docker-compose.staging.yml up -d ``` From 5e105eb8f3c3c1b231717511852600466d36280c Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 27 Mar 2025 20:30:15 -0300 Subject: [PATCH 08/28] solution for ssl certificate --- deploy/README.md | 12 ++++++++++-- deploy/nginx.staging.conf | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index d2bf502e5..3f3e21ff7 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -13,8 +13,16 @@ $ sudo cp nginx.staging.conf /etc/nginx/sites-available/default $ sudo systemctl restart nginx ``` -3. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. -4. Deploy using Docker Compose with file `docker-compose.staging.yml`. +3. Remember to set up SSL certificates (currently, self-signed ones are enough): +``` +$ sudo mkdir -p /etc/nginx/ssl +% sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout /etc/nginx/ssl/selfsigned.key \ + -out /etc/nginx/ssl/selfsigned.crt \ + -subj "/CN=*.serlo-staging.dev" +``` +4. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. +5. Deploy using Docker Compose with file `docker-compose.staging.yml`. ```console $ docker compose -f docker-compose.staging.yml up -d ``` diff --git a/deploy/nginx.staging.conf b/deploy/nginx.staging.conf index 0b5fe684e..e38e4e766 100644 --- a/deploy/nginx.staging.conf +++ b/deploy/nginx.staging.conf @@ -1,7 +1,11 @@ server { listen 80; + listen 443 ssl; server_name serlo-staging.dev; + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + location / { proxy_pass http://localhost:3001; proxy_set_header Host $host; @@ -13,8 +17,12 @@ server { server { listen 80; + listen 443 ssl; server_name kratos.serlo-staging.dev; + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + location / { proxy_pass http://localhost:4433; proxy_set_header Host $host; From f312de320d122e0a3a23c4688d3cf7dc634dd665 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 27 Mar 2025 21:01:41 -0300 Subject: [PATCH 09/28] Remove unnecessary file and adapt kratos config for staging --- deploy/docker-compose.staging.yml | 4 +- deploy/kratos/config.staging.yml | 2 +- deploy/kratos/config.yml | 191 ------------------------------ 3 files changed, 3 insertions(+), 194 deletions(-) delete mode 100644 deploy/kratos/config.yml diff --git a/deploy/docker-compose.staging.yml b/deploy/docker-compose.staging.yml index 8c393ec33..83c4d018c 100644 --- a/deploy/docker-compose.staging.yml +++ b/deploy/docker-compose.staging.yml @@ -42,12 +42,12 @@ services: depends_on: - postgres - kratos-migrate - command: serve -c /etc/config/kratos/config.yml --dev --watch-courier + command: serve -c /etc/config/kratos/config.staging.yml --watch-courier volumes: - ./kratos:/etc/config/kratos kratos-migrate: image: oryd/kratos:v1.3.0 - command: -c /etc/config/kratos/config.yml migrate sql -e --yes + command: -c /etc/config/kratos/config.staging.yml migrate sql -e --yes depends_on: - postgres volumes: diff --git a/deploy/kratos/config.staging.yml b/deploy/kratos/config.staging.yml index c0437dfab..750c61b7e 100644 --- a/deploy/kratos/config.staging.yml +++ b/deploy/kratos/config.staging.yml @@ -149,7 +149,7 @@ session: secrets: cookie: - - PLACEHOLDER + - PLACEHOLDERPLACEHOLDER identity: default_schema_id: default diff --git a/deploy/kratos/config.yml b/deploy/kratos/config.yml deleted file mode 100644 index aa93adbb6..000000000 --- a/deploy/kratos/config.yml +++ /dev/null @@ -1,191 +0,0 @@ -# TODO: remove when config.staging.yml has the correct values - -version: v1.0.0 - -dev: true - -dsn: postgres://serlo:secret@postgres:5432/kratos?sslmode=disable&max_conns=20&max_idle_conns=4 - -serve: - public: - base_url: http://localhost:4433/ - admin: - base_url: http://localhost:4434/ - -selfservice: - default_browser_return_url: http://localhost:3000/ - - methods: - password: - enabled: true - config: - haveibeenpwned_enabled: false - min_password_length: 6 - identifier_similarity_check_enabled: false - link: - enabled: true - oidc: - enabled: true - config: - providers: - - id: nbp - provider: generic - client_id: serlo - client_secret: H8t6WKWtGwFjqNfuAjqxrwCfsdznMAfj - issuer_url: http://nbp:11111/realms/master - mapper_url: file:///etc/config/kratos/user_mapper.jsonnet - - id: vidis - provider: generic - client_id: serlo - client_secret: H8t6WKWtGwFjqNfuAjqxrwCfsdznMAfj - issuer_url: http://vidis:11112/realms/master - mapper_url: file:///etc/config/kratos/user_mapper.jsonnet - - flows: - error: - ui_url: http://localhost:3000/auth/error - logout: - after: - default_browser_return_url: http://localhost:3000/auth/login - login: - ui_url: http://localhost:3000/auth/login - lifespan: 10m - after: - password: - hooks: - - hook: require_verified_address - - hook: web_hook - config: - url: http://host.docker.internal:3001/kratos/updateLastLogin - method: POST - body: file:///etc/config/kratos/identity_id.jsonnet - response: - ignore: true - oidc: - default_browser_return_url: http://localhost:3000/auth/login - - registration: - ui_url: http://localhost:3000/auth/registration - after: - hooks: - - hook: web_hook - config: - url: http://host.docker.internal:3001/kratos/register - method: POST - can_interrupt: true - body: file:///etc/config/kratos/identity_id.jsonnet - auth: - type: api_key - config: - name: x-kratos-key - value: api.serlo.org-kratos-secret - in: header - - hook: web_hook - config: - url: https://.api.mailchimp.com/3.0/lists//members - method: POST - body: file:///etc/config/kratos/add_to_newsletter.jsonnet - can_interrupt: false - response: - ignore: true - auth: - type: basic_auth - config: - user: serlo - password: PUT_HERE_YOUR_MAILCHIMP_API_KEY - - oidc: - hooks: - - hook: web_hook - config: - url: http://host.docker.internal:3001/kratos/register - method: POST - body: file:///etc/config/kratos/identity_id.jsonnet - auth: - type: api_key - config: - name: x-kratos-key - value: api.serlo.org-kratos-secret - in: header - - hook: web_hook - config: - url: https://.api.mailchimp.com/3.0/lists//members - method: POST - body: file:///etc/config/kratos/add_to_newsletter.jsonnet - can_interrupt: false - response: - ignore: true - auth: - type: basic_auth - config: - user: serlo - password: PUT_HERE_YOUR_MAILCHIMP_API_KEY - - hook: session - default_browser_return_url: http://localhost:3000/auth/login - verification: - enabled: true - use: link - ui_url: http://localhost:3000/auth/verification - recovery: - enabled: true - use: link - ui_url: http://localhost:3000/auth/recovery - - settings: - ui_url: http://localhost:3000/auth/settings -log: - level: debug - format: text - leak_sensitive_values: true - -secrets: - cookie: - - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE - cipher: - - 32-LONG-SECRET-NOT-SECURE-AT-ALL - -ciphers: - algorithm: xchacha20-poly1305 - -hashers: - algorithm: bcrypt - bcrypt: - cost: 8 - -identity: - default_schema_id: default - schemas: - - id: default - url: file:///etc/config/kratos/identity.schema.json - -courier: - smtp: - connection_uri: smtps://test:secret@mailslurper:1025/?skip_ssl_verify=true - template_override_path: /etc/config/kratos/email-templates/ - templates: - verification: - valid: - email: - subject: http://host.docker.internal:3000/api/.ory/mail-templates/verification/valid/email.subject.gotmpl - body: - html: http://host.docker.internal:3000/api/.ory/mail-templates/verification/valid/email.body.gotmpl - plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/verification/valid/email.body.plaintext.gotmpl - invalid: - email: - subject: http://host.docker.internal:3000/api/.ory/mail-templates/verification/invalid/email.subject.gotmpl - body: - html: http://host.docker.internal:3000/api/.ory/mail-templates/verification/invalid/email.body.gotmpl - plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/verification/invalid/email.body.plaintext.gotmpl - recovery: - valid: - email: - subject: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/valid/email.subject.gotmpl - body: - html: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/valid/email.body.gotmpl - plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/valid/email.body.plaintext.gotmpl - invalid: - email: - subject: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/invalid/email.subject.gotmpl - body: - html: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/invalid/email.body.gotmpl - plaintext: http://host.docker.internal:3000/api/.ory/mail-templates/recovery/invalid/email.body.plaintext.gotmpl From ae5b1dc9b2b67da7ca09eec13abe105e225c8ff9 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 27 Mar 2025 21:54:10 -0300 Subject: [PATCH 10/28] format --- deploy/docker-compose.staging.yml | 2 +- deploy/kratos/config.staging.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/docker-compose.staging.yml b/deploy/docker-compose.staging.yml index 83c4d018c..98de3980b 100644 --- a/deploy/docker-compose.staging.yml +++ b/deploy/docker-compose.staging.yml @@ -64,4 +64,4 @@ services: volumes: postgres_data: - mysql_data: \ No newline at end of file + mysql_data: diff --git a/deploy/kratos/config.staging.yml b/deploy/kratos/config.staging.yml index 750c61b7e..f581586f2 100644 --- a/deploy/kratos/config.staging.yml +++ b/deploy/kratos/config.staging.yml @@ -95,7 +95,7 @@ selfservice: config: url: https://api.serlo-staging.dev/kratos/register method: POST - body: file:///etc/config/kratos/identity_id.jsonnet + body: file:///etc/config/kratos/identity_id.jsonnet auth: type: api_key config: @@ -106,7 +106,7 @@ selfservice: config: url: https://PLACEHOLDER method: POST - body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet can_interrupt: false response: ignore: true From 391d07b93f3d82b86a11058383eb9cff9db5ff69 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 27 Mar 2025 22:13:40 -0300 Subject: [PATCH 11/28] Add instructions for set staging db up --- deploy/README.md | 16 ++++++++++++ deploy/dbsetup.sh | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100755 deploy/dbsetup.sh diff --git a/deploy/README.md b/deploy/README.md index 3f3e21ff7..fb32e04da 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,19 +1,25 @@ # Deployment Instructions with Docker Compose ## Requirements + - Docker - Nginx - Git +- GCloud CLI +- Gsutil ## Steps for the Staging deployment 1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org` 2. Set up Nginx on the host machine using configuration file `nginx.staging.conf`. + ```console $ sudo cp nginx.staging.conf /etc/nginx/sites-available/default $ sudo systemctl restart nginx ``` + 3. Remember to set up SSL certificates (currently, self-signed ones are enough): + ``` $ sudo mkdir -p /etc/nginx/ssl % sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ @@ -21,8 +27,18 @@ $ sudo mkdir -p /etc/nginx/ssl -out /etc/nginx/ssl/selfsigned.crt \ -subj "/CN=*.serlo-staging.dev" ``` + 4. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. 5. Deploy using Docker Compose with file `docker-compose.staging.yml`. + ```console $ docker compose -f docker-compose.staging.yml up -d ``` + +6. Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account + 1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one + 2. Put the key in a file `staging_service_account_key.json` in the home directory + 3. `echo $GCLOUD_SERVICE_ACCOUNT_KEY > $HOME/staging_service_account_key.json` + 4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file /tmp/service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. + 5. Run `./dbsetup.sh` + 6. Set cron tab to run the dbsetup script every night at 2 am. diff --git a/deploy/dbsetup.sh b/deploy/dbsetup.sh new file mode 100755 index 000000000..564e906a2 --- /dev/null +++ b/deploy/dbsetup.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +set -e + +mysql_connect="docker compose -f docker-compose.staging.yml exec mysql mysql --user=serlo --password=secret" + +echo "wait for mysql database to be ready" +until $mysql_connect -e "SHOW DATABASES" >/dev/null 2>/dev/null; do + echo "could not find mysql server - retry in 10 seconds" + sleep 10 +done + +newest_dump_uri=$(gsutil ls -l gs://anonymous-dump | grep dump | sort -rk 2 | head -n 1 | awk '{ print $3 }') +[ -z "$newest_dump_uri" ] && { + echo "no database dump available in anonymous db dump bucket" + exit 1 +} + +newest_dump=$(basename $newest_dump_uri) +[ -f "/tmp/$newest_dump" ] && exit 0 + +gsutil cp $newest_dump_uri "/tmp/$newest_dump" +echo "downloaded newest dump $newest_dump" +unzip -o "/tmp/$newest_dump" -d /tmp || { + echo "unzip of dump file failed" + exit 1 +} + +echo "Recreating serlo database" + +docker compose -f docker-compose.staging.yml cp /tmp/mysql.sql mysql:/tmp/mysql.sql +docker compose -f docker-compose.staging.yml cp /tmp/user.csv mysql:/tmp/user.csv + +mysql $mysql_connect -e "DROP DATABASE serlo" +mysql $mysql_connect -e "CREATE DATABASE serlo" +mysql $mysql_connect serlo <"/tmp/mysql.sql" || { + echo "import of dump failed" + exit 1 +} +mysql $mysql_connect -e "LOAD DATA LOCAL INFILE '/tmp/user.csv' INTO TABLE user FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' IGNORE 1 ROWS;" serlo || { + echo "import of dump failed" + exit 1 +} +mysql $mysql_connect serlo -e "UPDATE user SET description = NULL WHERE description = 'NULL'" + +echo "imported serlo database dump $newest_dump" + +echo "Recreating kratos database" + +docker compose -f docker-compose.staging.yml cp /tmp/kratos.sql postgres:/tmp/kratos.sql + +postgres_connect="docker compose -f docker-compose.staging.yml exec postgres psql --user=serlo kratos " +psql $postgres_connect -c "DROP SCHEMA public CASCADE;" +psql $postgres_connect -c "CREATE SCHEMA public;" +psql $postgres_connect -c "GRANT ALL ON SCHEMA public TO serlo;" +psql $postgres_connect Date: Thu, 27 Mar 2025 22:42:50 -0300 Subject: [PATCH 12/28] Correct script and minor changes --- deploy/README.md | 3 ++- deploy/dbsetup.sh | 19 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index fb32e04da..32f0d19c2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -7,6 +7,7 @@ - Git - GCloud CLI - Gsutil +- unzip ## Steps for the Staging deployment @@ -39,6 +40,6 @@ $ docker compose -f docker-compose.staging.yml up -d 1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one 2. Put the key in a file `staging_service_account_key.json` in the home directory 3. `echo $GCLOUD_SERVICE_ACCOUNT_KEY > $HOME/staging_service_account_key.json` - 4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file /tmp/service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. + 4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file ~/staging_service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. 5. Run `./dbsetup.sh` 6. Set cron tab to run the dbsetup script every night at 2 am. diff --git a/deploy/dbsetup.sh b/deploy/dbsetup.sh index 564e906a2..862d833bf 100755 --- a/deploy/dbsetup.sh +++ b/deploy/dbsetup.sh @@ -17,7 +17,6 @@ newest_dump_uri=$(gsutil ls -l gs://anonymous-dump | grep dump | sort -rk 2 | he } newest_dump=$(basename $newest_dump_uri) -[ -f "/tmp/$newest_dump" ] && exit 0 gsutil cp $newest_dump_uri "/tmp/$newest_dump" echo "downloaded newest dump $newest_dump" @@ -31,17 +30,17 @@ echo "Recreating serlo database" docker compose -f docker-compose.staging.yml cp /tmp/mysql.sql mysql:/tmp/mysql.sql docker compose -f docker-compose.staging.yml cp /tmp/user.csv mysql:/tmp/user.csv -mysql $mysql_connect -e "DROP DATABASE serlo" -mysql $mysql_connect -e "CREATE DATABASE serlo" -mysql $mysql_connect serlo <"/tmp/mysql.sql" || { +$mysql_connect -e "DROP DATABASE serlo" +$mysql_connect -e "CREATE DATABASE serlo" +$mysql_connect serlo <"/tmp/mysql.sql" || { echo "import of dump failed" exit 1 } -mysql $mysql_connect -e "LOAD DATA LOCAL INFILE '/tmp/user.csv' INTO TABLE user FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' IGNORE 1 ROWS;" serlo || { +$mysql_connect -e "LOAD DATA LOCAL INFILE '/tmp/user.csv' INTO TABLE user FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' IGNORE 1 ROWS;" serlo || { echo "import of dump failed" exit 1 } -mysql $mysql_connect serlo -e "UPDATE user SET description = NULL WHERE description = 'NULL'" +$mysql_connect serlo -e "UPDATE user SET description = NULL WHERE description = 'NULL'" echo "imported serlo database dump $newest_dump" @@ -50,10 +49,10 @@ echo "Recreating kratos database" docker compose -f docker-compose.staging.yml cp /tmp/kratos.sql postgres:/tmp/kratos.sql postgres_connect="docker compose -f docker-compose.staging.yml exec postgres psql --user=serlo kratos " -psql $postgres_connect -c "DROP SCHEMA public CASCADE;" -psql $postgres_connect -c "CREATE SCHEMA public;" -psql $postgres_connect -c "GRANT ALL ON SCHEMA public TO serlo;" -psql $postgres_connect Date: Thu, 27 Mar 2025 23:07:51 -0300 Subject: [PATCH 13/28] fix loading of file and tty for staging db setup --- deploy/dbsetup.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/dbsetup.sh b/deploy/dbsetup.sh index 862d833bf..b52ea3e2a 100755 --- a/deploy/dbsetup.sh +++ b/deploy/dbsetup.sh @@ -2,7 +2,7 @@ set -e -mysql_connect="docker compose -f docker-compose.staging.yml exec mysql mysql --user=serlo --password=secret" +mysql_connect="docker compose -f docker-compose.staging.yml exec -T mysql mysql --user=serlo --password=secret" echo "wait for mysql database to be ready" until $mysql_connect -e "SHOW DATABASES" >/dev/null 2>/dev/null; do @@ -36,8 +36,10 @@ $mysql_connect serlo <"/tmp/mysql.sql" || { echo "import of dump failed" exit 1 } -$mysql_connect -e "LOAD DATA LOCAL INFILE '/tmp/user.csv' INTO TABLE user FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' IGNORE 1 ROWS;" serlo || { - echo "import of dump failed" + +docker compose -f docker-compose.staging.yml exec -T mysql mysql --user=root --password=secret -e "SET GLOBAL local_infile = 1" +$mysql_connect --local-infile=1 -e "LOAD DATA LOCAL INFILE '/tmp/user.csv' INTO TABLE user FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' IGNORE 1 ROWS;" serlo || { + echo "import of users failed" exit 1 } $mysql_connect serlo -e "UPDATE user SET description = NULL WHERE description = 'NULL'" @@ -48,7 +50,7 @@ echo "Recreating kratos database" docker compose -f docker-compose.staging.yml cp /tmp/kratos.sql postgres:/tmp/kratos.sql -postgres_connect="docker compose -f docker-compose.staging.yml exec postgres psql --user=serlo kratos " +postgres_connect="docker compose -f docker-compose.staging.yml exec -T postgres psql --user=serlo kratos " $postgres_connect -c "DROP SCHEMA public CASCADE;" $postgres_connect -c "CREATE SCHEMA public;" $postgres_connect -c "GRANT ALL ON SCHEMA public TO serlo;" From 274118db54a1a023a23421b64cbc0ef8d7aa03c3 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 27 Mar 2025 23:40:46 -0300 Subject: [PATCH 14/28] specify subdomain for api --- deploy/README.md | 2 +- deploy/nginx.staging.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 32f0d19c2..84685bf4d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -41,5 +41,5 @@ $ docker compose -f docker-compose.staging.yml up -d 2. Put the key in a file `staging_service_account_key.json` in the home directory 3. `echo $GCLOUD_SERVICE_ACCOUNT_KEY > $HOME/staging_service_account_key.json` 4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file ~/staging_service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. - 5. Run `./dbsetup.sh` + 5. Run `./dbsetup.sh` in host 6. Set cron tab to run the dbsetup script every night at 2 am. diff --git a/deploy/nginx.staging.conf b/deploy/nginx.staging.conf index e38e4e766..59fadf6cc 100644 --- a/deploy/nginx.staging.conf +++ b/deploy/nginx.staging.conf @@ -1,7 +1,7 @@ server { listen 80; listen 443 ssl; - server_name serlo-staging.dev; + server_name api.serlo-staging.dev; ssl_certificate /etc/nginx/ssl/selfsigned.crt; ssl_certificate_key /etc/nginx/ssl/selfsigned.key; From 60d8db4f14780543137bd2bd860ca715f7612b99 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Fri, 28 Mar 2025 11:27:16 -0300 Subject: [PATCH 15/28] modularize deployment files --- deploy/README.md | 15 +++++---------- deploy/{.staging.env => staging/.env} | 0 deploy/{ => staging}/dbsetup.sh | 0 .../docker-compose.yml} | 8 ++++---- .../nginx.default.conf} | 0 5 files changed, 9 insertions(+), 14 deletions(-) rename deploy/{.staging.env => staging/.env} (100%) rename deploy/{ => staging}/dbsetup.sh (100%) rename deploy/{docker-compose.staging.yml => staging/docker-compose.yml} (92%) rename deploy/{nginx.staging.conf => staging/nginx.default.conf} (100%) diff --git a/deploy/README.md b/deploy/README.md index 84685bf4d..b3ba6c6b3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -11,11 +11,11 @@ ## Steps for the Staging deployment -1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org` -2. Set up Nginx on the host machine using configuration file `nginx.staging.conf`. +1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org/deploy` +2. Set up Nginx on the host machine using configuration file `staging/nginx.default.conf`. ```console -$ sudo cp nginx.staging.conf /etc/nginx/sites-available/default +$ sudo cp staging/nginx.default.conf /etc/nginx/sites-available/default $ sudo systemctl restart nginx ``` @@ -29,13 +29,8 @@ $ sudo mkdir -p /etc/nginx/ssl -subj "/CN=*.serlo-staging.dev" ``` -4. Be sure the values at `.staging.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. -5. Deploy using Docker Compose with file `docker-compose.staging.yml`. - -```console -$ docker compose -f docker-compose.staging.yml up -d -``` - +4. Be sure the values at `staging/.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. +5. Deploy using Docker Compose with file `staging/docker-compose.yml`. 6. Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account 1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one 2. Put the key in a file `staging_service_account_key.json` in the home directory diff --git a/deploy/.staging.env b/deploy/staging/.env similarity index 100% rename from deploy/.staging.env rename to deploy/staging/.env diff --git a/deploy/dbsetup.sh b/deploy/staging/dbsetup.sh similarity index 100% rename from deploy/dbsetup.sh rename to deploy/staging/dbsetup.sh diff --git a/deploy/docker-compose.staging.yml b/deploy/staging/docker-compose.yml similarity index 92% rename from deploy/docker-compose.staging.yml rename to deploy/staging/docker-compose.yml index 98de3980b..0ff033205 100644 --- a/deploy/docker-compose.staging.yml +++ b/deploy/staging/docker-compose.yml @@ -3,7 +3,7 @@ services: image: ghcr.io/serlo/api.serlo.org/server:staging pull_policy: always env_file: - - .staging.env + - .env ports: - '3001:3001' # TODO best practice: healthcheck @@ -14,7 +14,7 @@ services: image: ghcr.io/serlo/api.serlo.org/swr-queue-worker:staging pull_policy: always env_file: - - .staging.env + - .env # TODO: ports needed? ports: - '3000:3000' @@ -44,14 +44,14 @@ services: - kratos-migrate command: serve -c /etc/config/kratos/config.staging.yml --watch-courier volumes: - - ./kratos:/etc/config/kratos + - ../kratos:/etc/config/kratos kratos-migrate: image: oryd/kratos:v1.3.0 command: -c /etc/config/kratos/config.staging.yml migrate sql -e --yes depends_on: - postgres volumes: - - ./kratos:/etc/config/kratos + - ../kratos:/etc/config/kratos restart: on-failure postgres: image: postgres:13 diff --git a/deploy/nginx.staging.conf b/deploy/staging/nginx.default.conf similarity index 100% rename from deploy/nginx.staging.conf rename to deploy/staging/nginx.default.conf From 665775421d1f5f8fddd4cb0e712d873d7824e321 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Sat, 29 Mar 2025 08:20:09 -0300 Subject: [PATCH 16/28] Add basic files for production deploy --- deploy/README.md | 40 ++++-- deploy/kratos/config.production.yml | 191 +++++++++++++++++++++++++++ deploy/production/.env | 32 +++++ deploy/production/docker-compose.yml | 117 ++++++++++++++++ deploy/production/nginx.default.conf | 50 +++++++ 5 files changed, 420 insertions(+), 10 deletions(-) create mode 100644 deploy/kratos/config.production.yml create mode 100644 deploy/production/.env create mode 100644 deploy/production/docker-compose.yml create mode 100644 deploy/production/nginx.default.conf diff --git a/deploy/README.md b/deploy/README.md index b3ba6c6b3..94c966301 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -9,32 +9,52 @@ - Gsutil - unzip -## Steps for the Staging deployment +## Steps for the deployment -1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org/deploy` -2. Set up Nginx on the host machine using configuration file `staging/nginx.default.conf`. +1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org/` +2. `cd staging/` or `cd production/` depending on your enviroment. +3. Set up Nginx on the host machine using configuration file `nginx.default.conf`. ```console -$ sudo cp staging/nginx.default.conf /etc/nginx/sites-available/default +$ sudo cp nginx.default.conf /etc/nginx/sites-available/default # alternatively use the command `ln` $ sudo systemctl restart nginx ``` -3. Remember to set up SSL certificates (currently, self-signed ones are enough): +3. Set up SSL certificates (currently, self-signed ones are enough): ``` $ sudo mkdir -p /etc/nginx/ssl -% sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ +$ sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout /etc/nginx/ssl/selfsigned.key \ -out /etc/nginx/ssl/selfsigned.crt \ - -subj "/CN=*.serlo-staging.dev" + # uncomment what apply + # -subj "/CN=*.serlo-staging.dev" + # -subj "/CN=*.serlo.org" ``` -4. Be sure the values at `staging/.env` and `kratos/config.staging.yml` (change the values with "PLACEHODER") are correct. -5. Deploy using Docker Compose with file `staging/docker-compose.yml`. -6. Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account +4. Be sure the values at corresponding `.env` and `kratos/config.staging.yml` or `kratos/config.production.yml` (change the values with "PLACEHODER") are correct. +5. Deploy using Docker Compose. + +## Additional steps for STAGING + +### Serlo DB Setup +You need to fill up the database with data and set the cronjob for that for every night. + +Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account 1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one 2. Put the key in a file `staging_service_account_key.json` in the home directory 3. `echo $GCLOUD_SERVICE_ACCOUNT_KEY > $HOME/staging_service_account_key.json` 4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file ~/staging_service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. 5. Run `./dbsetup.sh` in host 6. Set cron tab to run the dbsetup script every night at 2 am. + +### DB Migration Cronjob +TODO + +## Additional steps for PRODUCTION + +### Serlo DB Dump +You need to set up a cronjob for doing mysql and postgres dump every night + +### Rocket Chat DB Dump +You need to set up a cronjob for doing the db dump of rocket chat every night diff --git a/deploy/kratos/config.production.yml b/deploy/kratos/config.production.yml new file mode 100644 index 000000000..ffa6e2a81 --- /dev/null +++ b/deploy/kratos/config.production.yml @@ -0,0 +1,191 @@ +dsn: postgres://serlo:secret@postgres:5432/kratos?sslmode=disable&max_conns=20&max_idle_conns=4 + +serve: + public: + base_url: https://kratos.serlo.org + request_log: + disable_for_health: true + admin: + request_log: + disable_for_health: true + +selfservice: + default_browser_return_url: https://serlo.org/ + allowed_return_urls: + # TODO: try with wildcard later + - https://fr.serlo.org/ + - https://hi.serlo.org/ + - https://de.serlo.org/ + - https://ta.serlo.org/ + - https://en.serlo.org/ + - https://es.serlo.org/ + methods: + password: + enabled: true + config: + haveibeenpwned_enabled: false + link: + enabled: true + config: + base_url: https://serlo.org/api/.ory/ + oidc: + enabled: true + config: + base_redirect_uri: https://serlo.org/api/.ory/ + providers: + - id: nbp + provider: generic + client_id: PLACEHOLDER + client_secret: PLACEHOLDER + issuer_url: https://aai.demo.meinbildungsraum.de/realms/nbp-aai + mapper_url: file:///etc/config/kratos/user_mapper.jsonnet + - id: vidis + provider: generic + client_id: PLACEHOLDER + client_secret: PLACEHOLDER + issuer_url: http://PLACEHOLDER + mapper_url: file:///etc/config/kratos/vidis_user_mapper.jsonnet + + flows: + error: + ui_url: https://serlo.org/auth/error + + settings: + ui_url: https://serlo.org/auth/settings + privileged_session_max_age: 15m + + recovery: + enabled: true + use: link + ui_url: https://serlo.org/auth/recovery + + verification: + enabled: true + use: link + ui_url: https://serlo.org/auth/verification + + logout: + after: + default_browser_return_url: https://serlo.org/auth/login + + login: + ui_url: https://serlo.org/auth/login + lifespan: 10m + after: + password: + hooks: + - hook: require_verified_address + - hook: web_hook + config: + url: https://api.serlo.org/kratos/updateLastLogin + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + response: + ignore: true + oidc: + default_browser_return_url: https://serlo.org/auth/login + + registration: + enable_legacy_one_step: true + lifespan: 10m + ui_url: https://serlo.org/auth/registration + after: + hooks: + - hook: web_hook + config: + url: https://api.serlo.org/kratos/register + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: PLACEHOLDER + in: header + - hook: web_hook + config: + url: https://PLACEHOLDER + method: POST + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PLACEHOLDER + oidc: + default_browser_return_url: https://serlo.org/auth/login + hooks: + - hook: web_hook + config: + url: https://api.serlo.org/kratos/register + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: PLACEHOLDER + in: header + - hook: session + - hook: web_hook + config: + url: https://PLACEHOLDER + method: POST + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PLACEHOLDER + +session: + lifespan: 720h + +secrets: + cookie: + - PLACEHOLDERPLACEHOLDER + +identity: + default_schema_id: default + schemas: + - id: default + url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtp://PLACEHOLDER + from_name: Serlo + from_address: no-reply@mail.serlo.org + templates: + verification: + valid: + email: + subject: http://serlo.org/api/.ory/mail-templates/verification/valid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/verification/valid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/verification/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.body.plaintext.gotmpl + recovery: + valid: + email: + subject: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.body.plaintext.gotmpl diff --git a/deploy/production/.env b/deploy/production/.env new file mode 100644 index 000000000..167ca2ced --- /dev/null +++ b/deploy/production/.env @@ -0,0 +1,32 @@ +# This a template. Replace the values with the actual ones in the environment. + +ENVIRONMENT=production +GOOGLE_SPREADSHEET_API_ACTIVE_DONORS=1qpyC0XzvTcKT6EISywvqESX3A0MwQoFDE8p-Bll4hps +GOOGLE_SPREADSHEET_API_MOTIVATION=142gKytX3rMT25DiIwQvhTLZrjZlojzZ4APIvwRuR4tU +GOOGLE_SPREADSHEET_API_SECRET=my-secret +LOG_LEVEL=ERROR +MAILCHIMP_API_KEY=secret-us5 +METADATA_API_VERSION=1.0.0 +MYSQL_URI=mysql://root:secret@mysql:3306/serlo?timezone=+00:00 +REDIS_URL=redis://redis:6379 +ROCKET_CHAT_API_USER_ID=an-user-id +ROCKET_CHAT_API_AUTH_TOKEN=an-auth-token +ROCKET_CHAT_URL=https://community.serlo.org/ +SERLO_ORG_DATABASE_LAYER_HOST=127.0.0.1:8080 +SERLO_ORG_SECRET=serlo.org-secret +# Set the following value to `empty` to disable caching +CACHE_TYPE=redis + +SERVER_HYDRA_HOST=http://hydra:4445 +SERVER_KRATOS_PUBLIC_HOST=http://kratos:4433 +SERVER_KRATOS_ADMIN_HOST=http://kratos:4434 +SERVER_KRATOS_SECRET=api.serlo.org-kratos-secret +SERVER_KRATOS_DB_URI=postgres://serlo:secret@postgres:5432/kratos?sslmode=disable\&max_conns=20\&max_idle_conns=4 +SERVER_SERLO_CLOUDFLARE_WORKER_SECRET=api.serlo.org-playground-secret +SERVER_SERLO_CACHE_WORKER_SECRET=api.serlo.org-cache-worker-secret +SERVER_SERLO_NOTIFICATION_EMAIL_SERVICE_SECRET=api.serlo.org-notification-email-service-secret +SERVER_SERLO_EDITOR_TESTING_SECRET=api.serlo.org-serlo-editor-testing-secret +SERVER_SWR_QUEUE_DASHBOARD_PASSWORD=secret +SERVER_SWR_QUEUE_DASHBOARD_USERNAME=secret + +OPENAI_API_KEY="" diff --git a/deploy/production/docker-compose.yml b/deploy/production/docker-compose.yml new file mode 100644 index 000000000..aaf4e9067 --- /dev/null +++ b/deploy/production/docker-compose.yml @@ -0,0 +1,117 @@ +services: + api: + image: ghcr.io/serlo/api.serlo.org/server:production + pull_policy: always + env_file: + - .env + ports: + - '3001:3001' + # TODO best practice: healthcheck + depends_on: + - mysql + - redis + swr-queue-worker: + image: ghcr.io/serlo/api.serlo.org/swr-queue-worker:production + pull_policy: always + env_file: + - .env + # TODO: ports needed? + ports: + - '3000:3000' + # TODO best practice: healthcheck + depends_on: + - mysql + - redis + redis: + image: redis:7 + mysql: + image: mysql:8 + environment: + - MYSQL_ROOT_PASSWORD=secret + - MYSQL_DATABASE=serlo + - MYSQL_USER=serlo + - MYSQL_PASSWORD=secret + volumes: + - mysql_data:/var/lib/mysql + kratos: + image: oryd/kratos:v1.3.0 + ports: + - '4433:4433' # public + environment: + - DSN=postgres://serlo:secret@postgres:5432/kratos?sslmode=disable + depends_on: + - postgres + - kratos-migrate + command: serve -c /etc/config/kratos/config.production.yml --watch-courier + volumes: + - ../kratos:/etc/config/kratos + kratos-migrate: + image: oryd/kratos:v1.3.0 + command: -c /etc/config/kratos/config.production.yml migrate sql -e --yes + depends_on: + - postgres + volumes: + - ../kratos:/etc/config/kratos + restart: on-failure + postgres: + image: postgres:13 + environment: + - POSTGRES_USER=serlo + - POSTGRES_PASSWORD=secret + - POSTGRES_DB=kratos + volumes: + - postgres_data:/var/lib/postgresql/data + hydra: + image: oryd/hydra:v2.2.0 + ports: + # why both? + - '4444:4444' + - '4445:4445' + command: serve all --dev + volumes: + # using sqlite, maybe postgres would be better + - hydra_data:/var/lib/sqlite + environment: + # - LOG_LEVEL=debug + - LOG_LEAK_SENSITIVE_VALUES=true + - OAUTH2_EXPOSE_INTERNAL_ERRORS=1 + - URLS_SELF_ISSUER=http://localhost:4444 + - URLS_LOGIN=http://localhost:3000/auth/oauth/login + - URLS_LOGOUT=http://localhost:3000/auth/oauth/logout + - URLS_CONSENT=http://localhost:3000/auth/oauth/consent + - DSN=memory + - SECRETS_SYSTEM=youReallyNeedToChangeThis + - OIDC_SUBJECT_IDENTIFIERS_ENABLED=public,pairwise + - OIDC_SUBJECT_IDENTIFIERS_PAIRWISE_SALT=youReallyNeedToChangeThis + rocketchat: + image: registry.rocket.chat/rocketchat/rocket.chat:7.4.0 + restart: on-failure + environment: + MONGO_URL: 'mongodb://mongodb:27017/rocketchat?replicaSet=rs0' + MONGO_OPLOG_URL: 'mongodb://mongodb:27017/local?replicaSet=rs0' + ROOT_URL: http://localhost:3030 + PORT: '3030' + DEPLOY_METHOD: docker + depends_on: + - mongodb + ports: + - '3030:3030' + mongodb: + image: docker.io/bitnami/mongodb:6.0 + restart: on-failure + volumes: + - mongodb_data:/bitnami/mongodb + environment: + MONGODB_REPLICA_SET_MODE: primary + MONGODB_REPLICA_SET_NAME: rs0 + MONGODB_PORT_NUMBER: '27017' + MONGODB_INITIAL_PRIMARY_HOST: mongodb + MONGODB_INITIAL_PRIMARY_PORT_NUMBER: '27017' + MONGODB_ADVERTISED_HOSTNAME: mongodb + MONGODB_ENABLE_JOURNAL: 'true' + ALLOW_EMPTY_PASSWORD: 'yes' +volumes: + mongodb_data: + hydra_data: + postgres_data: + mysql_data: diff --git a/deploy/production/nginx.default.conf b/deploy/production/nginx.default.conf new file mode 100644 index 000000000..fdacb7823 --- /dev/null +++ b/deploy/production/nginx.default.conf @@ -0,0 +1,50 @@ +server { + listen 80; + listen 443 ssl; + server_name api.serlo.org; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + location / { + proxy_pass http://localhost:3001; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + listen 443 ssl; + server_name kratos.serlo.org; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + location / { + proxy_pass http://localhost:4433; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + listen 443 ssl; + server_name community.serlo.org; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + location / { + proxy_pass http://localhost:3030; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} From 09d80caf202547a87ea07e3b9ca6e294c8911af2 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Mon, 31 Mar 2025 22:02:19 -0300 Subject: [PATCH 17/28] minos precisions in docs --- deploy/README.md | 52 +++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 94c966301..fdbdb00fd 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -11,50 +11,52 @@ ## Steps for the deployment -1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org/` +1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org/deploy/` 2. `cd staging/` or `cd production/` depending on your enviroment. 3. Set up Nginx on the host machine using configuration file `nginx.default.conf`. - -```console -$ sudo cp nginx.default.conf /etc/nginx/sites-available/default # alternatively use the command `ln` -$ sudo systemctl restart nginx -``` - -3. Set up SSL certificates (currently, self-signed ones are enough): - -``` -$ sudo mkdir -p /etc/nginx/ssl -$ sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ - -keyout /etc/nginx/ssl/selfsigned.key \ - -out /etc/nginx/ssl/selfsigned.crt \ - # uncomment what apply - # -subj "/CN=*.serlo-staging.dev" - # -subj "/CN=*.serlo.org" -``` - + 1. First you need to set up SSL certificates (currently, self-signed ones are enough): + ```console + $ sudo mkdir -p /etc/nginx/ssl + $ sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout /etc/nginx/ssl/selfsigned.key \ + -out /etc/nginx/ssl/selfsigned.crt \ + # uncomment what apply + # -subj "/CN=*.serlo-staging.dev" + # -subj "/CN=*.serlo.org" + ``` + 2. Then configure the routes: + ```console + $ sudo cp nginx.default.conf /etc/nginx/sites-available/default # alternatively use the command `ln` + $ sudo systemctl restart nginx + ``` 4. Be sure the values at corresponding `.env` and `kratos/config.staging.yml` or `kratos/config.production.yml` (change the values with "PLACEHODER") are correct. 5. Deploy using Docker Compose. ## Additional steps for STAGING ### Serlo DB Setup + You need to fill up the database with data and set the cronjob for that for every night. Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account - 1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one - 2. Put the key in a file `staging_service_account_key.json` in the home directory - 3. `echo $GCLOUD_SERVICE_ACCOUNT_KEY > $HOME/staging_service_account_key.json` - 4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file ~/staging_service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. - 5. Run `./dbsetup.sh` in host - 6. Set cron tab to run the dbsetup script every night at 2 am. + +1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one +2. Put the key in a file `staging_service_account_key.json` in the home directory +3. `echo $GCLOUD_SERVICE_ACCOUNT_KEY > $HOME/staging_service_account_key.json` +4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file ~/staging_service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. +5. Run `./dbsetup.sh` in host +6. Set cron tab to run the dbsetup script every night at 2 am. ### DB Migration Cronjob + TODO ## Additional steps for PRODUCTION ### Serlo DB Dump + You need to set up a cronjob for doing mysql and postgres dump every night ### Rocket Chat DB Dump + You need to set up a cronjob for doing the db dump of rocket chat every night From 2b3abb72f746eeaa503161651b3d92faa61e6cfd Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Mon, 31 Mar 2025 22:02:37 -0300 Subject: [PATCH 18/28] fix script for dbsetup --- deploy/staging/dbsetup.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/staging/dbsetup.sh b/deploy/staging/dbsetup.sh index b52ea3e2a..c444f5da0 100755 --- a/deploy/staging/dbsetup.sh +++ b/deploy/staging/dbsetup.sh @@ -2,7 +2,7 @@ set -e -mysql_connect="docker compose -f docker-compose.staging.yml exec -T mysql mysql --user=serlo --password=secret" +mysql_connect="docker compose exec -T mysql mysql --user=serlo --password=secret" echo "wait for mysql database to be ready" until $mysql_connect -e "SHOW DATABASES" >/dev/null 2>/dev/null; do @@ -27,8 +27,8 @@ unzip -o "/tmp/$newest_dump" -d /tmp || { echo "Recreating serlo database" -docker compose -f docker-compose.staging.yml cp /tmp/mysql.sql mysql:/tmp/mysql.sql -docker compose -f docker-compose.staging.yml cp /tmp/user.csv mysql:/tmp/user.csv +docker compose cp /tmp/mysql.sql mysql:/tmp/mysql.sql +docker compose cp /tmp/user.csv mysql:/tmp/user.csv $mysql_connect -e "DROP DATABASE serlo" $mysql_connect -e "CREATE DATABASE serlo" @@ -37,7 +37,7 @@ $mysql_connect serlo <"/tmp/mysql.sql" || { exit 1 } -docker compose -f docker-compose.staging.yml exec -T mysql mysql --user=root --password=secret -e "SET GLOBAL local_infile = 1" +docker compose exec -T mysql mysql --user=root --password=secret -e "SET GLOBAL local_infile = 1" $mysql_connect --local-infile=1 -e "LOAD DATA LOCAL INFILE '/tmp/user.csv' INTO TABLE user FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' IGNORE 1 ROWS;" serlo || { echo "import of users failed" exit 1 @@ -48,9 +48,9 @@ echo "imported serlo database dump $newest_dump" echo "Recreating kratos database" -docker compose -f docker-compose.staging.yml cp /tmp/kratos.sql postgres:/tmp/kratos.sql +docker compose cp /tmp/kratos.sql postgres:/tmp/kratos.sql -postgres_connect="docker compose -f docker-compose.staging.yml exec -T postgres psql --user=serlo kratos " +postgres_connect="docker compose exec -T postgres psql --user=serlo kratos " $postgres_connect -c "DROP SCHEMA public CASCADE;" $postgres_connect -c "CREATE SCHEMA public;" $postgres_connect -c "GRANT ALL ON SCHEMA public TO serlo;" From f9150ae50b9e84906945e01fe624c187cf5b109a Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Tue, 1 Apr 2025 11:02:15 -0300 Subject: [PATCH 19/28] minor documentation --- deploy/README.md | 1 + deploy/production/nginx.default.conf | 2 ++ deploy/staging/nginx.default.conf | 2 ++ 3 files changed, 5 insertions(+) diff --git a/deploy/README.md b/deploy/README.md index fdbdb00fd..83929a365 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -31,6 +31,7 @@ ``` 4. Be sure the values at corresponding `.env` and `kratos/config.staging.yml` or `kratos/config.production.yml` (change the values with "PLACEHODER") are correct. 5. Deploy using Docker Compose. +6. Set the DNS accordingly. At the server, remember set the firewall rules to allow http and https. ## Additional steps for STAGING diff --git a/deploy/production/nginx.default.conf b/deploy/production/nginx.default.conf index fdacb7823..febb592c9 100644 --- a/deploy/production/nginx.default.conf +++ b/deploy/production/nginx.default.conf @@ -1,3 +1,5 @@ +# TODO: block all requests that don't match to a host + server { listen 80; listen 443 ssl; diff --git a/deploy/staging/nginx.default.conf b/deploy/staging/nginx.default.conf index 59fadf6cc..c987b29c2 100644 --- a/deploy/staging/nginx.default.conf +++ b/deploy/staging/nginx.default.conf @@ -1,3 +1,5 @@ +# TODO: block all requests that don't match to a host + server { listen 80; listen 443 ssl; From be81c93a8d664deabbbfa8b5069eaeaf9573ef67 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Tue, 1 Apr 2025 21:20:01 -0300 Subject: [PATCH 20/28] protect sensitive files from being commited --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 301c7d7ab..a69d1145f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# May contain sensitive information +user.csv +mysql.sql +kratos.sql +temp.sql +dump-*.zip + # Local enmeshed connector /enmeshed/logs /enmeshed/config.json From 5e0bc4f212cac225e1a88ab4040bbcc515edee1d Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Tue, 1 Apr 2025 21:53:01 -0300 Subject: [PATCH 21/28] feat: add script for dumping anonymized dbs --- deploy/README.md | 17 +++++++---- deploy/production/dbdump.sh | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) create mode 100755 deploy/production/dbdump.sh diff --git a/deploy/README.md b/deploy/README.md index 83929a365..4b376f917 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -39,14 +39,13 @@ You need to fill up the database with data and set the cronjob for that for every night. -Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account +Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account. 1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one 2. Put the key in a file `staging_service_account_key.json` in the home directory -3. `echo $GCLOUD_SERVICE_ACCOUNT_KEY > $HOME/staging_service_account_key.json` -4. `gcloud auth activate-service-account ${GCLOUD_SERVICE_ACCOUNT_NAME} --key-file ~/staging_service_account_key.json` Replace GCLOUD_SERVICE_ACCOUNT_NAME with the email of the service account. -5. Run `./dbsetup.sh` in host -6. Set cron tab to run the dbsetup script every night at 2 am. +3. `gcloud auth activate-service-account --key-file ~/staging_service_account_key.json` +4. Run `./dbsetup.sh` in host +5. Set cron tab to run the dbsetup script every night at 2 am. ### DB Migration Cronjob @@ -56,7 +55,13 @@ TODO ### Serlo DB Dump -You need to set up a cronjob for doing mysql and postgres dump every night +You need to set up the cronjob for dumping the database for staging. + +Set up the Gsutil. You need the credentials of a service account in order that the script runs correctly. + +1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one +2. Put the key in a file `production_service_account_key.json` in the home directory +3. Set cron tab to run the dbsetup script every night at 1 am. ### Rocket Chat DB Dump diff --git a/deploy/production/dbdump.sh b/deploy/production/dbdump.sh new file mode 100755 index 000000000..972628259 --- /dev/null +++ b/deploy/production/dbdump.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +set -e + +mysql_cmd="docker compose exec -T mysql mysql --user=serlo --password=secret" + +echo "dump serlo.org database - start" + +echo "dump legacy serlo database schema" +mysql_dump_cmd="docker compose exec -T mysql mysqldump --user=serlo --password=secret --lock-tables=false" + +$mysql_dump_cmd --no-data --add-drop-database --databases serlo >mysql.sql + +$mysql_dump_cmd --no-create-info --lock-tables=false --add-locks --ignore-table=serlo.user serlo >>mysql.sql + +$mysql_cmd --batch -e "SELECT id, CONCAT(@rn:=@rn+1, '@localhost') AS email, username, '8a534960a8a4c8e348150a0ae3c7f4b857bfead4f02c8cbf0d' AS password, logins, date, CONCAT(@rn:=@rn+1, '') AS token, last_login, description FROM user, (select @rn:=2) r;" serlo >user.csv + +echo "dump kratos identities data" + +echo "dumping the data from the kratos database" +docker compose exec postgres pg_dump --user=serlo kratos >temp.sql + +echo "creating another container to manipulate and anonymize the data" +docker run --name temp_postgres -e POSTGRES_PASSWORD=secret -e POSTGRES_PASSWORD=password -e POSTGRES_DB=kratos -e PGPASSWORD=secret -v ./temp.sql:/temp.sql -d postgres:13 + +echo "waiting for the postgres container to be ready" +if ! timeout 60s sh -c 'until docker exec temp_postgres pg_isready -U postgres; do sleep 1; done'; then + echo "Postgres container did not become ready within 1 minute." + exit 1 +fi + +temp_postgres_cmd="docker exec temp_postgres psql -h localhost -U postgres --quiet -c" +$temp_postgres_cmd "CREATE user serlo;" +$temp_postgres_cmd "GRANT ALL PRIVILEGES ON DATABASE kratos TO serlo;" + +docker exec -i temp_postgres psql -h localhost -U serlo -d kratos > 'interest' != 'teacher';" +$temp_postgres_cmd "UPDATE identity_credentials SET config = '{\"hashed_password\": \"\$sha1\$pf=e1NBTFR9e1BBU1NXT1JEfQ==\$YTQwYzEwY2ZlNA==\$hTlqikjjSFoK43S4V7+t8CyMvw0=\"}';" +$temp_postgres_cmd "UPDATE identity_verifiable_addresses SET value = CONCAT(identity_id, '@localhost');" +$temp_postgres_cmd "UPDATE identity_recovery_addresses SET value = CONCAT(identity_id, '@localhost');" +$temp_postgres_cmd "UPDATE identity_credential_identifiers SET identifier = CONCAT(ic.identity_id, '@localhost') FROM (select id, identity_id FROM identity_credentials) AS ic where ic.id = identity_credential_id and identifier LIKE '%@%';" +$temp_postgres_cmd "TRUNCATE sessions, continuity_containers, courier_messages, identity_verification_codes, identity_recovery_codes, identity_recovery_tokens, identity_verification_tokens, selfservice_errors, selfservice_login_flows, selfservice_recovery_flows, selfservice_registration_flows, selfservice_settings_flows, selfservice_verification_flows, session_devices, session_token_exchanges, identity_login_codes CASCADE;" +docker exec temp_postgres pg_dump -h localhost -U serlo kratos >kratos.sql + +echo "compressing database dump" +day=$(date -I) +zip "dump-$day".zip mysql.sql user.csv kratos.sql + +gcloud auth activate-service-account --key-file=~/production_service-account.json +gsutil cp dump-$day gs://anonymous-dump +echo "latest dump uploaded" + +echo "removing temporary files and containers" +rm -f mysql.sql user.csv kratos.sql dump-$day.zip temp.sql +docker rm -f temp_postgres >/dev/null 2>&1 + +echo "dump of serlo.org database - end" From 4855ec17a5673c9ea2855a7cc0fd6afb64551e5e Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Tue, 1 Apr 2025 21:53:38 -0300 Subject: [PATCH 22/28] minor adjust to make accessing db in postgres easier --- deploy/production/docker-compose.yml | 1 + deploy/staging/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/deploy/production/docker-compose.yml b/deploy/production/docker-compose.yml index aaf4e9067..60fc8fe3a 100644 --- a/deploy/production/docker-compose.yml +++ b/deploy/production/docker-compose.yml @@ -59,6 +59,7 @@ services: - POSTGRES_USER=serlo - POSTGRES_PASSWORD=secret - POSTGRES_DB=kratos + - PGPASSWORD=secret volumes: - postgres_data:/var/lib/postgresql/data hydra: diff --git a/deploy/staging/docker-compose.yml b/deploy/staging/docker-compose.yml index 0ff033205..af4018d6d 100644 --- a/deploy/staging/docker-compose.yml +++ b/deploy/staging/docker-compose.yml @@ -59,6 +59,7 @@ services: - POSTGRES_USER=serlo - POSTGRES_PASSWORD=secret - POSTGRES_DB=kratos + - PGPASSWORD=secret volumes: - postgres_data:/var/lib/postgresql/data From 7416bc43714ac3d5b99347cb110d972bd7cc736b Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Wed, 2 Apr 2025 09:31:57 -0300 Subject: [PATCH 23/28] block all resquests from unknown hosts --- deploy/production/nginx.default.conf | 13 +++++++++++-- deploy/staging/nginx.default.conf | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/deploy/production/nginx.default.conf b/deploy/production/nginx.default.conf index febb592c9..9b856d6e7 100644 --- a/deploy/production/nginx.default.conf +++ b/deploy/production/nginx.default.conf @@ -1,5 +1,3 @@ -# TODO: block all requests that don't match to a host - server { listen 80; listen 443 ssl; @@ -50,3 +48,14 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } } + +server { + listen 80 default_server; + listen 443 ssl default_server; + server_name _; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + return 444; +} diff --git a/deploy/staging/nginx.default.conf b/deploy/staging/nginx.default.conf index c987b29c2..82a685856 100644 --- a/deploy/staging/nginx.default.conf +++ b/deploy/staging/nginx.default.conf @@ -1,5 +1,3 @@ -# TODO: block all requests that don't match to a host - server { listen 80; listen 443 ssl; @@ -33,3 +31,14 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } } + +server { + listen 80 default_server; + listen 443 ssl default_server; + server_name _; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + return 444; +} From 416d38808231eddac51bf4f88b87f47a2a52955c Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Wed, 2 Apr 2025 10:20:34 -0300 Subject: [PATCH 24/28] add instruction for db-migration --- deploy/README.md | 12 +++++++++++- deploy/production/docker-compose.yml | 25 +++++++++++++++++++++++++ deploy/staging/docker-compose.yml | 24 ++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/deploy/README.md b/deploy/README.md index 4b376f917..c8fcb9e20 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -49,7 +49,10 @@ Set up the Gsutil. You need to authenticate and may use a key of the appropriate ### DB Migration Cronjob -TODO +Add a crontab in host with the following command (replace the missing values) for 3 am. +``` +docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network staging-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER +``` ## Additional steps for PRODUCTION @@ -66,3 +69,10 @@ Set up the Gsutil. You need the credentials of a service account in order that t ### Rocket Chat DB Dump You need to set up a cronjob for doing the db dump of rocket chat every night + +### DB Migration + +In case of db migration, run the following command in host (replace the missing values). +``` +docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network production-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER +``` \ No newline at end of file diff --git a/deploy/production/docker-compose.yml b/deploy/production/docker-compose.yml index 60fc8fe3a..8e57a7c62 100644 --- a/deploy/production/docker-compose.yml +++ b/deploy/production/docker-compose.yml @@ -10,6 +10,8 @@ services: depends_on: - mysql - redis + networks: + - production-network swr-queue-worker: image: ghcr.io/serlo/api.serlo.org/swr-queue-worker:production pull_policy: always @@ -22,8 +24,12 @@ services: depends_on: - mysql - redis + networks: + - production-network redis: image: redis:7 + networks: + - production-network mysql: image: mysql:8 environment: @@ -33,6 +39,8 @@ services: - MYSQL_PASSWORD=secret volumes: - mysql_data:/var/lib/mysql + networks: + - production-network kratos: image: oryd/kratos:v1.3.0 ports: @@ -45,6 +53,8 @@ services: command: serve -c /etc/config/kratos/config.production.yml --watch-courier volumes: - ../kratos:/etc/config/kratos + networks: + - production-network kratos-migrate: image: oryd/kratos:v1.3.0 command: -c /etc/config/kratos/config.production.yml migrate sql -e --yes @@ -53,6 +63,8 @@ services: volumes: - ../kratos:/etc/config/kratos restart: on-failure + networks: + - production-network postgres: image: postgres:13 environment: @@ -62,6 +74,8 @@ services: - PGPASSWORD=secret volumes: - postgres_data:/var/lib/postgresql/data + networks: + - production-network hydra: image: oryd/hydra:v2.2.0 ports: @@ -84,6 +98,8 @@ services: - SECRETS_SYSTEM=youReallyNeedToChangeThis - OIDC_SUBJECT_IDENTIFIERS_ENABLED=public,pairwise - OIDC_SUBJECT_IDENTIFIERS_PAIRWISE_SALT=youReallyNeedToChangeThis + networks: + - production-network rocketchat: image: registry.rocket.chat/rocketchat/rocket.chat:7.4.0 restart: on-failure @@ -97,6 +113,8 @@ services: - mongodb ports: - '3030:3030' + networks: + - production-network mongodb: image: docker.io/bitnami/mongodb:6.0 restart: on-failure @@ -111,6 +129,13 @@ services: MONGODB_ADVERTISED_HOSTNAME: mongodb MONGODB_ENABLE_JOURNAL: 'true' ALLOW_EMPTY_PASSWORD: 'yes' + networks: + - production-network + +networks: + production-network: + driver: bridge + volumes: mongodb_data: hydra_data: diff --git a/deploy/staging/docker-compose.yml b/deploy/staging/docker-compose.yml index af4018d6d..10badc0ad 100644 --- a/deploy/staging/docker-compose.yml +++ b/deploy/staging/docker-compose.yml @@ -10,6 +10,9 @@ services: depends_on: - mysql - redis + networks: + - staging-network + swr-queue-worker: image: ghcr.io/serlo/api.serlo.org/swr-queue-worker:staging pull_policy: always @@ -22,8 +25,14 @@ services: depends_on: - mysql - redis + networks: + - staging-network + redis: image: redis:7 + networks: + - staging-network + mysql: image: mysql:8 environment: @@ -33,6 +42,9 @@ services: - MYSQL_PASSWORD=secret volumes: - mysql_data:/var/lib/mysql + networks: + - staging-network + kratos: image: oryd/kratos:v1.3.0 ports: @@ -45,6 +57,9 @@ services: command: serve -c /etc/config/kratos/config.staging.yml --watch-courier volumes: - ../kratos:/etc/config/kratos + networks: + - staging-network + kratos-migrate: image: oryd/kratos:v1.3.0 command: -c /etc/config/kratos/config.staging.yml migrate sql -e --yes @@ -53,6 +68,9 @@ services: volumes: - ../kratos:/etc/config/kratos restart: on-failure + networks: + - staging-network + postgres: image: postgres:13 environment: @@ -62,6 +80,12 @@ services: - PGPASSWORD=secret volumes: - postgres_data:/var/lib/postgresql/data + networks: + - staging-network + +networks: + staging-network: + driver: bridge volumes: postgres_data: From 778b41ea388ac7060ac719f62dbcbb0107db51fe Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Wed, 2 Apr 2025 10:51:27 -0300 Subject: [PATCH 25/28] Add missing env vars --- deploy/production/.env | 4 ++++ deploy/staging/.env | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/deploy/production/.env b/deploy/production/.env index 167ca2ced..7c37a3a08 100644 --- a/deploy/production/.env +++ b/deploy/production/.env @@ -29,4 +29,8 @@ SERVER_SERLO_EDITOR_TESTING_SECRET=api.serlo.org-serlo-editor-testing-secret SERVER_SWR_QUEUE_DASHBOARD_PASSWORD=secret SERVER_SWR_QUEUE_DASHBOARD_USERNAME=secret +SWR_QUEUE_WORKER_CONCURRENCY=2 +SWR_QUEUE_WORKER_DELAY=250 +CHECK_STALLED_JOBS_DELAY=600000 + OPENAI_API_KEY="" diff --git a/deploy/staging/.env b/deploy/staging/.env index 0a35edd32..b178eca95 100644 --- a/deploy/staging/.env +++ b/deploy/staging/.env @@ -29,6 +29,10 @@ SERVER_SERLO_EDITOR_TESTING_SECRET=api.serlo.org-serlo-editor-testing-secret SERVER_SWR_QUEUE_DASHBOARD_PASSWORD=secret SERVER_SWR_QUEUE_DASHBOARD_USERNAME=secret +SWR_QUEUE_WORKER_CONCURRENCY=1 +SWR_QUEUE_WORKER_DELAY=250 +CHECK_STALLED_JOBS_DELAY=600000 + ENMESHED_SERVER_HOST="http://enmeshed:8081/" ENMESHED_SERVER_SECRET="apiKey" ENMESHED_WEBHOOK_SECRET="webhookKey" From 6c3d5059be9c0ff89fbe12312bd3fb5b5c37e6c5 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:16:14 -0300 Subject: [PATCH 26/28] Add script for doing the rocket chat db dump and minor fix --- .gitignore | 2 +- deploy/README.md | 17 ++++++++++++++--- deploy/production/mongodbdump.sh | 22 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) create mode 100755 deploy/production/mongodbdump.sh diff --git a/.gitignore b/.gitignore index a69d1145f..07e5c7ee7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ user.csv mysql.sql kratos.sql temp.sql -dump-*.zip +dump-* # Local enmeshed connector /enmeshed/logs diff --git a/deploy/README.md b/deploy/README.md index c8fcb9e20..c94889afe 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -50,6 +50,7 @@ Set up the Gsutil. You need to authenticate and may use a key of the appropriate ### DB Migration Cronjob Add a crontab in host with the following command (replace the missing values) for 3 am. + ``` docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network staging-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER ``` @@ -64,15 +65,25 @@ Set up the Gsutil. You need the credentials of a service account in order that t 1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one 2. Put the key in a file `production_service_account_key.json` in the home directory -3. Set cron tab to run the dbsetup script every night at 1 am. +3. Set cron tab to run the dbdump script every night at 1 am. ### Rocket Chat DB Dump -You need to set up a cronjob for doing the db dump of rocket chat every night +In your first deployment, you will need to import the existing data into mongodb container. + +1. Download the dump from the corresponding bucket in the GC project 'production' +2. Run + ``` + $ docker compose cp dump-????.gz mongodb:/dump.gz + $ docker compose exec mongodb mongorestore --archive=dump.gz --gzip + ``` + +Now, set up a crontab to upload a backup of the data to the bucket at midnight, using the script `mongodbdump.sh`. ### DB Migration In case of db migration, run the following command in host (replace the missing values). + ``` docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network production-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER -``` \ No newline at end of file +``` diff --git a/deploy/production/mongodbdump.sh b/deploy/production/mongodbdump.sh new file mode 100755 index 000000000..d035074fc --- /dev/null +++ b/deploy/production/mongodbdump.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -e + +echo "rocket-chat mongodump - start" + +echo "cleaning up old dumps" +docker compose exec -u root mongodb rm /dump.gz || true + +echo "creating new dump" +docker compose exec -u root mongodb mongodump --archive=dump.gz --gzip +dumpname="dump-$(date -I).gz" +docker compose cp mongodb:/dump.gz "$dumpname" + +echo "uploading dump to GCS" +gsutil cp $dumpname gs://serlo-production-rocket-chat-mongodump + +echo "cleaning up" +docker compose exec -u root mongodb rm /dump.gz || true +rm $dumpname + +echo "rocket-chat mongodump - end" From 639193356711323f5665acd6ed1d1ee2f1aa0bf1 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Thu, 3 Apr 2025 12:10:58 -0300 Subject: [PATCH 27/28] minor instruction regarding importing dumps --- deploy/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deploy/README.md b/deploy/README.md index c94889afe..58b6ed8c3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -59,7 +59,11 @@ docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PL ### Serlo DB Dump -You need to set up the cronjob for dumping the database for staging. +In your first deployment, you will need to import the existing data into mysql and postgres containers. +Manually dump the production databases. Take a look at `staging/dbsetup.sh` for some inspiration on how to +import the data. + +Afterwards, You need to set up the cronjob for dumping the database for staging. Set up the Gsutil. You need the credentials of a service account in order that the script runs correctly. From 17b2b795171b4991b67ce43dc2261dc9db4955f9 Mon Sep 17 00:00:00 2001 From: hugotiburtino <45924645+hugotiburtino@users.noreply.github.com> Date: Tue, 8 Apr 2025 13:56:01 -0300 Subject: [PATCH 28/28] minor fix in documentation --- deploy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/README.md b/deploy/README.md index 58b6ed8c3..5494e94cc 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -52,7 +52,7 @@ Set up the Gsutil. You need to authenticate and may use a key of the appropriate Add a crontab in host with the following command (replace the missing values) for 3 am. ``` -docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network staging-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER +docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network staging_staging-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER ``` ## Additional steps for PRODUCTION