diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 10c3eba..ac49551 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -1,3 +1,8 @@ +# Production with host-mounted partitions (no Docker named volumes). +# Persistent data: /store/keep. Cache/tmp: /store/tmp. Subdirs use snake_case. +# Change base paths in this file if using a different partition. +# See docs/PRODUCTION-VM-PROVISIONING.md. + x-app: &app env_file: - .env.production @@ -17,23 +22,16 @@ x-app: &app - BUNDLE_DISABLE_LOCAL_BRANCH_CHECK=true - BUNDLE_BUNDLER_INJECT__GEM_PATH=/app/samvera/bundler.d volumes: - - node_modules:/app/samvera/hyrax-webapp/node_modules:cached - - uploads:/app/samvera/hyrax-webapp/public/uploads:cached - - assets:/app/samvera/hyrax-webapp/public/assets:cached - - cache:/app/samvera/hyrax-webapp/tmp/cache:cached - .:/app/samvera - -volumes: - assets: - cache: - db: - fcrepo: - node_modules: - redis: - solr: - uploads: - zk: - zoo: + - /store/tmp/node_modules:/app/samvera/hyrax-webapp/node_modules:cached + - /store/keep/public_uploads:/app/samvera/hyrax-webapp/public/uploads:cached + - /store/tmp/public_assets:/app/samvera/hyrax-webapp/public/assets:cached + - /store/tmp/cache:/app/samvera/hyrax-webapp/tmp/cache:cached + logging: + driver: "json-file" + options: + max-size: "50m" + max-file: "5" networks: internal: @@ -43,12 +41,17 @@ services: extends: file: hyrax-webapp/docker-compose.production.yml service: zoo + volumes: + - /store/keep/zoo_data:/data + - /store/keep/zoo_datalog:/datalog solr: image: ghcr.io/notch8/wvu_knapsack/solr:${TAG:-latest} extends: file: hyrax-webapp/docker-compose.production.yml service: solr + volumes: + - /store/keep/solr_data:/var/solr fcrepo: env_file: @@ -56,6 +59,8 @@ services: extends: file: hyrax-webapp/docker-compose.production.yml service: fcrepo + volumes: + - /store/keep/fcrepo_data:/data:cached db: env_file: @@ -63,6 +68,8 @@ services: extends: file: hyrax-webapp/docker-compose.production.yml service: db + volumes: + - /store/keep/db_data:/var/lib/postgresql/data web: <<: *app @@ -97,11 +104,11 @@ services: # Uncomment command to access container with out starting bin/worker. Useful for debugging or updating Gemfile.lock # command: sleep infinity volumes: - - node_modules:/app/samvera/hyrax-webapp/node_modules:cached - - uploads:/app/samvera/hyrax-webapp/public/uploads:cached - - assets:/app/samvera/hyrax-webapp/public/assets:cached - - cache:/app/samvera/hyrax-webapp/tmp/cache:cached - .:/app/samvera + - /store/tmp/node_modules:/app/samvera/hyrax-webapp/node_modules:cached + - /store/keep/public_uploads:/app/samvera/hyrax-webapp/public/uploads:cached + - /store/tmp/public_assets:/app/samvera/hyrax-webapp/public/assets:cached + - /store/tmp/cache:/app/samvera/hyrax-webapp/tmp/cache:cached check_volumes: <<: *app @@ -119,3 +126,5 @@ services: extends: file: hyrax-webapp/docker-compose.production.yml service: redis + volumes: + - /store/keep/redis_data:/data diff --git a/docs/PRODUCTION-VM-PROVISIONING.md b/docs/PRODUCTION-VM-PROVISIONING.md new file mode 100644 index 0000000..243d6ab --- /dev/null +++ b/docs/PRODUCTION-VM-PROVISIONING.md @@ -0,0 +1,233 @@ +# Production Deployment Walkthrough: WVU Hyku (wvu_knapsack) + +This guide is a **single end-to-end walkthrough** for standing up production on a new VM: what to do **first** after the VM is provisioned, then in order, and what to do **last**. + +--- + +## Overview + +- The app runs via **Docker Compose** with `docker-compose.production.yml` from the **root** of the knapsack repo. +- The repo uses a **Git submodule** for `hyrax-webapp` (the Hyku app). A plain `git clone` leaves that folder empty until you run `git submodule init` and `git submodule update`. +- Production uses **host bind mounts** (no Docker named volumes): `/store/keep` (persistent data) and `/store/tmp` (cache). Subdirectories use **snake_case**. You must create these directories on the host before starting the stack. +- Configuration is in **`.env.production`** (not in the repo; create it from `.env` and set production values). +- **Ansible** is the next step: it prepares the VM (Docker, Nginx, SSL, GHCR login). See [Step 1: Prepare the VM with Ansible](#step-1-prepare-the-vm-with-ansible). + +--- + +## Prerequisites + +- A **target VM** (or server) that you can reach via SSH. Ansible will install Docker, Nginx, and other packages on it. +- **Ansible** installed on the machine from which you run the playbook (your laptop or a control node). +- Network: the target VM will need access to the knapsack Git repo and to **GitHub Container Registry (ghcr.io)**; you need a GHCR token with at least `read:packages` (Ansible can configure login with it). + +--- + +## Step 1: Prepare the VM with Ansible + +Run the Ansible playbook to set up the server: Docker, Nginx, SSL, GHCR login, and optional users. Run this from your laptop or control node; it targets the VM. + +1. Read **`ansible/README.md`**. +2. Place required files in `ansible/files/` (SSL cert/key, Nginx config, optional deploy key). +3. Create and encrypt the GHCR token (e.g. `ansible/ghcr-token.vault.yml`). +4. Edit `ansible/inventory.ini` (target host, `ssh_users`, etc.). +5. Run: + ```bash + ansible-playbook -i ansible/inventory.ini ansible/playbook.yml --ask-vault-pass -e @ansible/ghcr-token.vault.yml + ``` + +Then continue with **Step 2** on that VM (clone the repo, submodule, reserved branch, host directories). + +--- + +## Step 2: Get the code on the server and create host directories + +Do this once per server (or per clone). + +### 1a. Clone the repository and populate the submodule + +A normal `git clone` does **not** fill the `hyrax-webapp` directory (it's a Git submodule). You must init and update it. + +```bash +git clone git@github.com:notch8/wvu_knapsack.git wvu_knapsack +cd wvu_knapsack + +git submodule init +git submodule update +``` + +**Future updates:** `git pull` then `git submodule update --init`. + +### 1b. Create the required Knapsack branch (reserved branch) + +The build expects a local branch named `required_for_knapsack_instances`: + +```bash +git remote add prime https://github.com/samvera-labs/hyku_knapsack # omit if already added +git fetch prime +git checkout prime/required_for_knapsack_instances +git switch -c required_for_knapsack_instances +``` + +You can switch back to `main` afterward; the branch just needs to exist. + +### 1c. Create the host directories for data and cache + +The production Compose file uses **host bind mounts** (no Docker volumes). Paths are FHS-style with **snake_case** subdirs. Create them before starting the stack: + +```bash +# Persistent data +sudo mkdir -p /store/keep/{zoo_data,zoo_datalog,solr_data,fcrepo_data,db_data,redis_data,public_uploads} + +# Cache and tmp +sudo mkdir -p /store/tmp/{node_modules,public_assets,cache} +``` + +If your data lives on a different partition, edit the base paths in the root `docker-compose.production.yml` (search for `/store/keep` and `/store/tmp`). Ensure the user that runs Docker can read/write these directories. + +**Path reference:** + +| Use | Host path (default) | Container path | +|-----|---------------------|----------------| +| Zookeeper | `/store/keep/zoo_data`, `zoo_datalog` | `/data`, `/datalog` | +| Solr | `/store/keep/solr_data` | `/var/solr` | +| Fedora | `/store/keep/fcrepo_data` | `/data` | +| PostgreSQL | `/store/keep/db_data` | `/var/lib/postgresql/data` | +| Redis | `/store/keep/redis_data` | `/data` | +| App uploads | `/store/keep/public_uploads` | `…/public/uploads` | +| App assets / cache | `/store/tmp/public_assets`, `cache`, `node_modules` | `…/public/assets`, `…/tmp/cache`, `…/node_modules` | + +Do **not** remove or comment out the **check_volumes** service in the Compose file; it runs once to fix permissions on these mounts and is required for `web` and `worker` to start. + +--- + +## Step 3: Create `.env.production` + +Production Compose reads **`.env.production`** (gitignored). Create it from the repo's `.env` and set production-safe values: + +```bash +cp .env .env.production +# Edit .env.production +``` + +**Set at least:** `SECRET_KEY_BASE` (e.g. `bin/rails secret`), `DB_PASSWORD`, `INITIAL_ADMIN_EMAIL`, `INITIAL_ADMIN_PASSWORD`, `NEGATIVE_CAPTCHA_SECRET`, `SOLR_ADMIN_PASSWORD`, `PASSENGER_APP_ENV=production`. For host/tenant: either multi-tenant (`HYKU_MULTITENANT=true` and set `HYKU_ROOT_HOST`, etc.) or single-tenant (`HYKU_MULTITENANT=false`, `HYKU_ROOT_HOST`). Recommended: `HYKU_SSL_CONFIGURED=true` and SMTP/contact email. Full variable list: [Hyku configuration](../hyrax-webapp/docs/configuration.md). + +--- + +## Step 4: Log in to GitHub Container Registry (GHCR) + +The stack pulls images from `ghcr.io`. Log in so the server can pull: + +```bash +echo YOUR_GHCR_PAT | docker login ghcr.io -u YOUR_GITHUB_USERNAME --password-stdin +``` + +(GHCR can also be configured via the Ansible playbook; see `ansible/README.md`.) + +--- + +## Step 5: Build and run the stack + +All commands from the **root of the knapsack repo** (where `docker-compose.production.yml` and `hyrax-webapp` are). Use **dotenv** with **`.env.production`** and set an alias so pulls and restarts use the same env and compose file. + +### Alias and production workflow + +Set an alias that loads `.env.production` and uses the production compose file. You need **dotenv** on the host (e.g. `gem install dotenv` or `sudo gem install dotenv`). Then use `TAG=` for reproducible image tags when pulling and starting. + +```bash +cd /store/keep/wvu_knapsack # or wherever you cloned the repo + +alias dc='dotenv -e .env.production docker-compose -f docker-compose.production.yml' + +# Pull images tagged with current commit (run after git pull) +TAG=`git rev-parse --short=8 HEAD` dc pull + +# Bring up the stack (scale workers as needed) +TAG=`git rev-parse --short=8 HEAD` dc up -d --scale worker=3 web worker +``` + +To make the alias persistent, add it to `~/.bashrc` (or your shell profile): + +```bash +echo "alias dc='dotenv -e .env.production docker-compose -f docker-compose.production.yml'" >> ~/.bashrc +``` + +**Note:** Other knapsacks (e.g. adventist_knapsack) may define extra worker services like `worker_aux`; scale and service names accordingly, e.g. `dc up -d --scale worker=3 --scale worker_aux=3 worker worker_aux web`. For wvu_knapsack the app services are **web** and **worker** only. + +### First run (build) + +The first time, build the images, then bring everything up: + +```bash +docker compose -f docker-compose.production.yml build +TAG=`git rev-parse --short=8 HEAD` dc up -d +``` + +The first run starts dependencies, runs migrations and seed via `initialize_app`, then starts web and worker. The app listens on port 3000. + +**Useful:** `dc logs -f` | `dc down`; after changing `.env.production`, `dc down` then `dc up -d` (and rebuild if needed). + +--- + +## Step 6: Reverse proxy and SSL + +Put **Nginx** (or another proxy) in front of the app and terminate SSL. Point the proxy at the host port that maps to the web service (e.g. `http://localhost:3000`). The Ansible playbook in Step 1 installs Nginx and SSL; adjust the Nginx config so the app upstream is that host:port. + +--- + +## Step 7 (last): Verify, logs, and support + +- **Verify:** Open your production URL (e.g. `https://hyku.lib.wvu.edu`) and confirm the app loads and you can sign in with the initial admin account. +- **Container logs:** `docker compose -f docker-compose.production.yml logs -f` (or per service). On disk: `/var/lib/docker/containers//-json.log`. The Compose file already sets app logging with `max-size: "50m"` and `max-file: "5"`. +- **Application logs (optional):** To keep Rails logs on the host when containers are down, add a bind mount to the web/worker volumes, e.g. `- /var/log/hyku:/app/samvera/hyrax-webapp/log`, create `/var/log/hyku` on the host, then e.g. `tail -f /var/log/hyku/production.log`. +- **Support:** For questions, contact your Notch8 project lead (e.g. April) or the team that provided this documentation. + +--- + +## Deploy script (hykudev) + +For **routine deploys to the hykudev (development) server**, use the script at **`bin/deploy-hykudev.sh`**. It updates submodules, brings down old containers, pulls images tagged with the current git rev, and brings up the web service. + +**Typical workflow:** + +1. Connect to WVU VPN. +2. SSH to the hykudev server: `hykudev.lib.wvu.edu` (e.g. with your username). +3. Switch to the deploy user: `sudo su - ansible`. +4. Go to the repo: `cd wvu_knapsack`. +5. Pull the branch you want: `git pull origin main` (or the branch name). +6. Run the deploy script: `./bin/deploy-hykudev.sh`. + +The script uses `.env.development` and `docker compose` (no `-f docker-compose.production.yml`). After it finishes, the app is at: + +- Admin tenant: https://hykudev-admin.lib.wvu.edu +- Default tenant: https://hykudev.lib.wvu.edu + +For production, use the steps in this walkthrough (build/up with `docker-compose.production.yml`); there is no production deploy script in this repo. + +--- + +## Troubleshooting: "solr failed to start" and check_volumes + +- The root `docker-compose.production.yml` **extends** `hyrax-webapp/docker-compose.production.yml`. Volume definitions come from the child unless overridden in the root. This repo's root file already overrides all data services to use host bind mounts; do **not** rely on editing only the file inside `hyrax-webapp/` to change volumes. +- Do **not** remove or comment out **check_volumes**. `web`, `worker`, and `initialize_app` depend on it; it also runs `chown` on the mounted app paths so the app user can write. If it's disabled or never completes, the stack will not start correctly. + +--- + +## Checklist + +- [ ] **Step 1:** Run Ansible playbook to prepare the VM (Docker, Nginx, SSL, GHCR login). See `ansible/README.md`. +- [ ] **Step 2:** On the VM: clone repo; `git submodule init` and `git submodule update`; create branch `required_for_knapsack_instances` from `prime/required_for_knapsack_instances`; create host dirs under `/store/keep` and `/store/tmp` (or your partition). +- [ ] **Step 3:** Create `.env.production` from `.env` and set production secrets and host/tenant settings. +- [ ] **Step 4:** `docker login ghcr.io` (or equivalent; may already be done by Ansible). +- [ ] **Step 5:** From repo root: `docker compose -f docker-compose.production.yml build` then `up -d`. +- [ ] **Step 6:** Configure reverse proxy and SSL to the app port (e.g. 3000). +- [ ] **Step 7:** Verify app at production URL; note where logs are; add optional app log mount if desired. + +--- + +## Reference + +- **Main README:** [README.md](../README.md) — submodule, reserved branch, development. +- **Hyku configuration:** [hyrax-webapp/docs/configuration.md](../hyrax-webapp/docs/configuration.md) — all environment variables. +- **Ansible:** [ansible/README.md](../ansible/README.md) — server prep. +- **Deploy script (hykudev):** [bin/deploy-hykudev.sh](../bin/deploy-hykudev.sh) — routine deploys to hykudev.lib.wvu.edu. +