experiment: How could it be to deploy using docker only (no kubernetes)

.gitignore

@@ -1,3 +1,10 @@
+# May contain sensitive information
+user.csv
+mysql.sql
+kratos.sql
+temp.sql
+dump-*
+
 # Local enmeshed connector
 /enmeshed/logs
 /enmeshed/config.json

deploy/README.md

@@ -0,0 +1,93 @@
+# Deployment Instructions with Docker Compose
+
+## Requirements
+
+- Docker
+- Nginx
+- Git
+- GCloud CLI
+- Gsutil
+- unzip
+
+## Steps for the deployment
+
+1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org/deploy/`
+2. `cd staging/` or `cd production/` depending on your enviroment.
+3. Set up Nginx on the host machine using configuration file `nginx.default.conf`.
+   1. First you need to set up SSL certificates (currently, self-signed ones are enough):
+   ```console
+   $ sudo mkdir -p /etc/nginx/ssl
+   $ sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+     -keyout /etc/nginx/ssl/selfsigned.key \
+     -out /etc/nginx/ssl/selfsigned.crt \
+      # uncomment what apply
+      # -subj "/CN=*.serlo-staging.dev"
+      # -subj "/CN=*.serlo.org"
+   ```
+   2. Then configure the routes:
+   ```console
+   $ sudo cp nginx.default.conf /etc/nginx/sites-available/default # alternatively use the command `ln`
+   $ sudo systemctl restart nginx
+   ```
+4. Be sure the values at corresponding `.env` and `kratos/config.staging.yml` or `kratos/config.production.yml` (change the values with "PLACEHODER") are correct.
+5. Deploy using Docker Compose.
+6. Set the DNS accordingly. At the server, remember set the firewall rules to allow http and https.
+
+## Additional steps for STAGING
+
+### Serlo DB Setup
+
+You need to fill up the database with data and set the cronjob for that for every night.
+
+Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account.
+
+1.  Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one
+2.  Put the key in a file `staging_service_account_key.json` in the home directory
+3.  `gcloud auth activate-service-account --key-file ~/staging_service_account_key.json`
+4.  Run `./dbsetup.sh` in host
+5.  Set cron tab to run the dbsetup script every night at 2 am.
+
+### DB Migration Cronjob
+
+Add a crontab in host with the following command (replace the missing values) for 3 am.
+
+```
+docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER"  --network staging_staging-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER
+```
+
+## Additional steps for PRODUCTION
+
+### Serlo DB Dump
+
+In your first deployment, you will need to import the existing data into mysql and postgres containers.
+Manually dump the production databases. Take a look at `staging/dbsetup.sh` for some inspiration on how to
+import the data.
+
+Afterwards, You need to set up the cronjob for dumping the database for staging.
+
+Set up the Gsutil. You need the credentials of a service account in order that the script runs correctly.
+
+1.  Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one
+2.  Put the key in a file `production_service_account_key.json` in the home directory
+3.  Set cron tab to run the dbdump script every night at 1 am.
+
+### Rocket Chat DB Dump
+
+In your first deployment, you will need to import the existing data into mongodb container.
+
+1. Download the dump from the corresponding bucket in the GC project 'production'
+2. Run
+   ```
+   $ docker compose cp dump-????.gz mongodb:/dump.gz
+   $ docker compose exec mongodb mongorestore --archive=dump.gz --gzip
+   ```
+
+Now, set up a crontab to upload a backup of the data to the bucket at midnight, using the script `mongodbdump.sh`.
+
+### DB Migration
+
+In case of db migration, run the following command in host (replace the missing values).
+
+```
+docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER"  --network production-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER
+```

deploy/kratos/config.production.yml

@@ -0,0 +1,191 @@
+dsn: postgres://serlo:secret@postgres:5432/kratos?sslmode=disable&max_conns=20&max_idle_conns=4
+
+serve:
+  public:
+    base_url: https://kratos.serlo.org
+    request_log:
+      disable_for_health: true
+  admin:
+    request_log:
+      disable_for_health: true
+
+selfservice:
+  default_browser_return_url: https://serlo.org/
+  allowed_return_urls:
+    # TODO: try with wildcard later
+    - https://fr.serlo.org/
+    - https://hi.serlo.org/
+    - https://de.serlo.org/
+    - https://ta.serlo.org/
+    - https://en.serlo.org/
+    - https://es.serlo.org/
+  methods:
+    password:
+      enabled: true
+      config:
+        haveibeenpwned_enabled: false
+    link:
+      enabled: true
+      config:
+        base_url: https://serlo.org/api/.ory/
+    oidc:
+      enabled: true
+      config:
+        base_redirect_uri: https://serlo.org/api/.ory/
+        providers:
+          - id: nbp
+            provider: generic
+            client_id: PLACEHOLDER
+            client_secret: PLACEHOLDER
+            issuer_url: https://aai.demo.meinbildungsraum.de/realms/nbp-aai
+            mapper_url: file:///etc/config/kratos/user_mapper.jsonnet
+          - id: vidis
+            provider: generic
+            client_id: PLACEHOLDER
+            client_secret: PLACEHOLDER
+            issuer_url: http://PLACEHOLDER
+            mapper_url: file:///etc/config/kratos/vidis_user_mapper.jsonnet
+
+  flows:
+    error:
+      ui_url: https://serlo.org/auth/error
+
+    settings:
+      ui_url: https://serlo.org/auth/settings
+      privileged_session_max_age: 15m
+
+    recovery:
+      enabled: true
+      use: link
+      ui_url: https://serlo.org/auth/recovery
+
+    verification:
+      enabled: true
+      use: link
+      ui_url: https://serlo.org/auth/verification
+
+    logout:
+      after:
+        default_browser_return_url: https://serlo.org/auth/login
+
+    login:
+      ui_url: https://serlo.org/auth/login
+      lifespan: 10m
+      after:
+        password:
+          hooks:
+            - hook: require_verified_address
+            - hook: web_hook
+              config:
+                url: https://api.serlo.org/kratos/updateLastLogin
+                method: POST
+                body: file:///etc/config/kratos/identity_id.jsonnet
+                response:
+                  ignore: true
+        oidc:
+          default_browser_return_url: https://serlo.org/auth/login
+
+    registration:
+      enable_legacy_one_step: true
+      lifespan: 10m
+      ui_url: https://serlo.org/auth/registration
+      after:
+        hooks:
+          - hook: web_hook
+            config:
+              url: https://api.serlo.org/kratos/register
+              method: POST
+              body: file:///etc/config/kratos/identity_id.jsonnet
+              auth:
+                type: api_key
+                config:
+                  name: x-kratos-key
+                  value: PLACEHOLDER
+                  in: header
+          - hook: web_hook
+            config:
+              url: https://PLACEHOLDER
+              method: POST
+              body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet
+              can_interrupt: false
+              response:
+                ignore: true
+              auth:
+                type: basic_auth
+                config:
+                  user: serlo
+                  password: PLACEHOLDER
+        oidc:
+          default_browser_return_url: https://serlo.org/auth/login
+          hooks:
+            - hook: web_hook
+              config:
+                url: https://api.serlo.org/kratos/register
+                method: POST
+                body: file:///etc/config/kratos/identity_id.jsonnet
+                auth:
+                  type: api_key
+                  config:
+                    name: x-kratos-key
+                    value: PLACEHOLDER
+                    in: header
+            - hook: session
+            - hook: web_hook
+              config:
+                url: https://PLACEHOLDER
+                method: POST
+                body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet
+                can_interrupt: false
+                response:
+                  ignore: true
+                auth:
+                  type: basic_auth
+                  config:
+                    user: serlo
+                    password: PLACEHOLDER
+
+session:
+  lifespan: 720h
+
+secrets:
+  cookie:
+    - PLACEHOLDERPLACEHOLDER
+
+identity:
+  default_schema_id: default
+  schemas:
+    - id: default
+      url: file:///etc/config/kratos/identity.schema.json
+
+courier:
+  smtp:
+    connection_uri: smtp://PLACEHOLDER
+    from_name: Serlo
+    from_address: [email protected]
+  templates:
+    verification:
+      valid:
+        email:
+          subject: http://serlo.org/api/.ory/mail-templates/verification/valid/email.subject.gotmpl
+          body:
+            html: http://serlo.org/api/.ory/mail-templates/verification/valid/email.body.gotmpl
+            plaintext: http://serlo.org/api/.ory/mail-templates/verification/valid/email.body.plaintext.gotmpl
+      invalid:
+        email:
+          subject: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.subject.gotmpl
+          body:
+            html: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.body.gotmpl
+            plaintext: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.body.plaintext.gotmpl
+    recovery:
+      valid:
+        email:
+          subject: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.subject.gotmpl
+          body:
+            html: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.body.gotmpl
+            plaintext: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.body.plaintext.gotmpl
+      invalid:
+        email:
+          subject: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.subject.gotmpl
+          body:
+            html: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.body.gotmpl
+            plaintext: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.body.plaintext.gotmpl