Simplify compose.yml (suggestion)

[dashohoxha]

I am not sure whether this is the right place to discuss this topic, but I think that compose.yml is much more complex that it needs to be. This is because it is being (ab)used as a scripting tool, to automate some steps. A better tool to implement automation would be bash scripting (the obvious choice, which is so obvious that nobody ever thinks about it, especially in the context of docker compose projects).

For example, to automate tinybird setup I would suggest a script like this:

setup-tinybird.sh

cmd_setup-tinybird_help() {
    cat <<_EOF
    setup-tinybird

        Automates the steps for Tinybird setup.  But it is not
        completely automated, requires manual intervention.

        Before starting, make sure you have an account
        with a workspace at: https://www.tinybird.co/

_EOF
}

cmd_setup-tinybird() {
    cat <<EOF

Before starting, make sure you have an account with a workspace
at: https://www.tinybird.co/

Press Ctrl-C to interrupt, or Enter to continue...

EOF
    read

    set -x
    get_the_repo
    build_the_image
    tinybird_login
    tinybird_sync
    tinybird_deploy
    get_tinybird_tokens
    update_config_files
    cleanup_and_restart
}

get_the_repo() {
    : Make a temporary directory and get the ghost-docker repo
    rm -rf tmp/
    mkdir tmp
    cd tmp/
    git clone https://github.com/TryGhost/ghost-docker.git .
    cd tinybird/
}

build_the_image() {
    : Build the docker image
    docker build -t tinybird-image .
}

tinybird_login() {
    : Authenticate the terminal
    docker run --rm -it \
           -v $(pwd)/tinybird_home:/home/tinybird \
           -v $(pwd)/tinybird_files:/data/tinybird \
           -w /home/tinybird \
           tinybird-image \
           /usr/local/bin/tinybird-login
}

tinybird_sync() {
    : Sync Tinybird files
    docker run --rm -it \
           -v $(pwd)/tinybird_files:/data/tinybird \
           ghost:6-alpine \
           bash -c 'cp -rf /var/lib/ghost/current/core/server/data/tinybird/* /data/tinybird/'
}

tinybird_deploy() {
    : Deploy Tinybird files to the workspace
    docker run --rm -it \
           -v $(pwd)/tinybird_home:/home/tinybird \
           -v $(pwd)/tinybird_files:/data/tinybird \
           -w /data/tinybird \
           tinybird-image \
           bash -c 'tb-wrapper --cloud deploy'
}

get_tinybird_tokens() {
    : Get the tokens
    docker run --rm -it \
           -v $(pwd)/tinybird_home:/home/tinybird \
           -v $(pwd)/tinybird_files:/data/tinybird \
           -w /home/tinybird \
           tinybird-image \
           get-tokens \
           > ../../env/tinybird-tokens.env
}

update_config_files() {
    : Read the tokens and update config files
    cd ../../env/
    source tinybird-tokens.env

    cat <<EOF > ghost-tinybird.env
tinybird__tracker__endpoint=https://${DOMAIN}/.ghost/analytics/api/v1/page_hit
tinybird__adminToken=${TINYBIRD_ADMIN_TOKEN}
tinybird__workspaceId=${TINYBIRD_WORKSPACE_ID}
tinybird__tracker__datasource=analytics_events
tinybird__stats__endpoint=${TINYBIRD_API_URL}
EOF
    
    cat <<EOF > analytics.env
NODE_ENV=production
PROXY_TARGET=${TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events
SALT_STORE_TYPE=${SALT_STORE_TYPE:-file}
SALT_STORE_FILE_PATH=/data/salts.json
TINYBIRD_TRACKER_TOKEN=${TINYBIRD_TRACKER_TOKEN}
LOG_LEVEL=debug
EOF

    : Enable the analytics profile
    cd ..
    echo 'COMPOSE_PROFILES=activitypub,analytics' > .env
}

cleanup_and_restart() {
    : Cleanup
    docker rmi tinybird-image
    rm -rf tmp/

    : Restart
    docker compose up -d
}

This would make obsolete the services tinybird-login, tinybird-sync, tinybird-deploy, etc.

In my implementation, compose.yml looks like this:

create_compose_yml() {
    local container_name=${DOMAIN//./-}

    # create compose.yml
    cat <<EOF > compose.yml
services:
  ghost:
    image: ghost:6-alpine
    container_name: $container_name
    restart: unless-stopped
    env_file:
      - env/ghost.env
      - env/ghost-tinybird.env
    volumes:
      - ./content:/var/lib/ghost/content

  activitypub:
    image: ghcr.io/tryghost/activitypub:1.2.2
    container_name: ${container_name}-activitypub
    restart: unless-stopped
    profiles: [activitypub]
    env_file: env/activitypub.env
    volumes:
      - ./content:/opt/activitypub/content

  traffic-analytics:
    image: ghost/traffic-analytics:1.0.204
    container_name: ${container_name}-analytics
    restart: unless-stopped
    env_file: env/analytics.env
    profiles: [analytics]
    volumes:
      - ./content/traffic_analytics:/data

networks:
  default:
    external: true
    name: $NETWORK
EOF
}

Some of the modifications that I have done are these:

• I have my own (external) reverse proxy and database containers (because I use them for other projects too)
• I am using env_file: to define all the env variables of a service (this makes compose.yml smaller)
• All the volumes are mounted to the local sub-directory ./content (this makes backup easier)

env files look like this:

env files

create_env_files() {
    # create ghost.env, if it does not exist
    local smtp_domain=${SMTP_DOMAIN:-${DOMAIN#*.}}
    local smtp_server=${SMTP_SERVER:-smtp.$smtp_domain}
    local from_address=${DOMAIN%%.*}@$smtp_domain
    mkdir -p env
    cat <<EOF > env/ghost.env
NODE_ENV=production
url=https://$DOMAIN

database__client=mysql
database__connection__host=$DBHOST
database__connection__user=$DBUSER
database__connection__password=$DBPASS
database__connection__database=$DBNAME

mail__transport=SMTP
mail__options__host=$smtp_server
mail__options__requireTLS=false
mail__from=$from_address

EOF
    
    cat <<'EOF' > env/ghost-tinybird.env
#tinybird__tracker__endpoint=https://${DOMAIN}/.ghost/analytics/api/v1/page_hit
#tinybird__adminToken=${TINYBIRD_ADMIN_TOKEN:-}
#tinybird__workspaceId=${TINYBIRD_WORKSPACE_ID:-}
#tinybird__tracker__datasource=analytics_events
#tinybird__stats__endpoint=${TINYBIRD_API_URL:-https://api.tinybird.co}
EOF
    
    cat <<EOF > env/activitypub.env
NODE_ENV=production
MYSQL_HOST=$DBHOST
MYSQL_USER=$DBUSER
MYSQL_PASSWORD=$DBPASS
MYSQL_DATABASE=${DBNAME}_activitypub
LOCAL_STORAGE_PATH=/opt/activitypub/content/images/activitypub
LOCAL_STORAGE_HOSTING_URL=https://${DOMAIN}/content/images/activitypub
EOF

    cat <<'EOF' > env/analytics.env
NODE_ENV=production
PROXY_TARGET=${TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events
SALT_STORE_TYPE=${SALT_STORE_TYPE:-file}
SALT_STORE_FILE_PATH=/data/salts.json
TINYBIRD_TRACKER_TOKEN=${TINYBIRD_TRACKER_TOKEN:-}
LOG_LEVEL=debug
EOF
}

---

By the way, tinybird analytics is not working for me. ActivityPub is also not working properly (for example mastodon.social cannot find my handle @index@gazeta.univlora.al). Maybe it is related to my reverse proxy configuration (NGINX), which looks like this:

nginx config

server {
    listen 443 ssl;
    server_name gazeta.univlora.al ;

    ssl_certificate     certs/gazeta.univlora.al/fullchain.pem;
    ssl_certificate_key certs/gazeta.univlora.al/privkey.pem;

    access_log /var/log/nginx/gazeta.univlora.al-access.log;
    error_log /var/log/nginx/gazeta.univlora.al-error.log error;
    
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
    add_header X-XSS-Protection "1; mode=block" always;
    add_header X-Content-Type-Options "nosniff" always;
    add_header Referrer-Policy "strict-origin-when-cross-origin" always;
    add_header Content-Security-Policy "frame-ancestors 'self'";

    include conf.d/proxy_params;
    proxy_set_header X-Forwarded-Host $host;

    #proxy_http_version 1.1;
    #proxy_set_header Upgrade $http_upgrade;
    #proxy_set_header Connection "upgrade";
    
    location / {
        proxy_pass http://gazeta-univlora-al:2368;
    }   
    
    location ^~ /.ghost/activitypub/ {
        proxy_pass http://gazeta-univlora-al-activitypub:8080;
    }   

    location ~ ^/\.well-known/(webfinger|nodeinfo) {
        proxy_pass http://gazeta-univlora-al-activitypub:8080;
    }   

    location ~ ^.*/\.ghost/analytics(?<suffix>.*)$ {
        proxy_pass http://gazeta-univlora-al-analytics:3000$suffix;
    }   
}