Deploy conversational chat to production

.github/workflows/deploy.yml

@@ -20,17 +20,17 @@ jobs:
       contents: read
     environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
     steps:
-      - name: Set CONFIG_ENV from Branch Name
+      - name: Set DEPLOY_ENV from Branch Name
         run: |
           if [[ $BRANCH == 'refs/heads/main' ]]; then
-            echo "CONFIG_ENV=production" >> $GITHUB_ENV
+            echo "DEPLOY_ENV=production" >> $GITHUB_ENV
           else
-            echo "CONFIG_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
+            echo "DEPLOY_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
           fi
         env:
           BRANCH: ${{ github.ref }}
       - name: Confirm deploy environment
-        run: echo "Deploying to '$CONFIG_ENV' environment"
+        run: echo "Deploying to '$DEPLOY_ENV' environment"
       - name: Set GitHub Deploy Key
         uses: webfactory/ssh-agent@v0.5.3
         with:
@@ -49,16 +49,13 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
           aws-region: us-east-1
-      - run: ln -s .tfvars/dc-api/samconfig.toml .
-      - run: ln -s .tfvars/dc-api/$CONFIG_ENV.parameters .
+      - run: ln -s .tfvars/dc-api/samconfig.${DEPLOY_ENV}.yaml .
       - run: make build
       - run: | 
           sam deploy \
             --no-confirm-changeset \
             --no-fail-on-empty-changeset \
-            --config-env $CONFIG_ENV \
-            --config-file ./samconfig.toml \
-            --parameter-overrides $(while IFS='=' read -r key value; do params+=" $key=$value"; done < ./$CONFIG_ENV.parameters && echo "$params HoneybadgerRevision=$HONEYBADGER_REVISION") \
+            --config-file ./samconfig.${DEPLOY_ENV}.yaml \
           | sed 's/\(Parameter overrides\s*\): .*/\1: ***** REDACTED *****/'
           exit ${PIPESTATUS[0]}
         env:

.github/workflows/test-node.yml

@@ -3,11 +3,11 @@ on:
   push:
     paths:
       - ".github/workflows/test-node.yml"
-      - "node/**"
+      - "api/**"
   workflow_dispatch:
 defaults:
   run:
-    working-directory: ./node
+    working-directory: ./api
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -20,7 +20,7 @@ jobs:
         with:
           node-version: 20.x
           cache: "npm"
-          cache-dependency-path: 'node/package-lock.json'
+          cache-dependency-path: 'api/package-lock.json'
       - run: npm ci
       - name: Check code style
         run: npm run lint && npm run prettier

.github/workflows/test-python.yml

@@ -14,7 +14,6 @@ jobs:
     env:
       AWS_ACCESS_KEY_ID: ci
       AWS_SECRET_ACCESS_KEY: ci
-      SKIP_LLM_REQUEST: 'True' 
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -27,5 +26,7 @@ jobs:
         run: ruff check .
       - name: Run tests
         run: |
-          coverage run --include='src/**/*' -m unittest
+          coverage run --include='src/**/*' -m pytest -m ""
           coverage report
+        env:
+          AWS_REGION: us-east-1

.github/workflows/validate-template.yml

@@ -16,13 +16,15 @@ jobs:
         - uses: actions/setup-python@v4
           with:
             python-version: '3.12' 
+        - name: Install cfn-lint
+          run: pip install cfn-lint
         - uses: aws-actions/setup-sam@v1
-        - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
-          run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
+        # - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
+        #   run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
         - uses: aws-actions/configure-aws-credentials@master
           with:
             role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
             aws-region: us-east-1
         - uses: actions/checkout@v3
         - name: Validate template
-          run: sam build && sam validate
+          run: make build && make validate

.gitignore

@@ -231,3 +231,5 @@ env.*.json
 *.parameters
 /schemas
 .sam-pids
+
+av-download/layers/

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
-cd node && npm run lint && npm run prettier && cd -
+cd api && npm run lint && npm run prettier && cd -
 cd chat/src && ruff check . && cd -

.tool-versions

@@ -1,4 +1,4 @@
 nodejs 20.15.0
 java corretto-19.0.1.10.1
-aws-sam-cli 1.107.0
+aws-sam-cli 1.135.0
 python 3.12.2

Makefile

@@ -5,92 +5,138 @@ ENV=dev
 SHELL := /bin/bash
 
 help:
-	echo "make build           | build the SAM project"
-	echo "make serve           | alias for serve-https"
-	echo "make clean           | remove all installed dependencies and build artifacts"
-	echo "make deps            | install all dependencies"
-	echo "make link            | create hard links to allow for hot reloading of a built project"
-	echo "make secrets         | symlink secrets files from ../tfvars"
-	echo "make start-with-step | run the SAM server locally with step function & download lambdas"
-	echo "make style           | run all style checks"
-	echo "make test            | run all tests"
-	echo "make cover           | run all tests with coverage"
-	echo "make env ENV=[env]   | activate env.\$$ENV.json file (default: dev)"
-	echo "make deps-node       | install node dependencies"
-	echo "make deps-python     | install python dependencies"
-	echo "make serve-http      | run the SAM server locally (HTTP on port 3000)"
-	echo "make serve-https     | run the SAM server locally (HTTPS on port 3002)"
-	echo "make style-node      | run node code style check"
-	echo "make style-python    | run python code style check"
-	echo "make test-node       | run node tests"
-	echo "make test-python     | run python tests"
-	echo "make cover-node      | run node tests with coverage"
-	echo "make cover-python    | run python tests with coverage"
-.aws-sam/build.toml: ./template.yaml node/package-lock.json node/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
-	sed -Ei.orig 's/^(\s+)#\*\s/\1/' template.yaml
-	sed -Ei.orig 's/^(\s+)#\*\s/\1/' chat/template.yaml
-	sam build --cached --parallel
-	mv template.yaml.orig template.yaml
-	mv chat/template.yaml.orig chat/template.yaml
+	echo "make build                 | build the SAM project"
+	echo "make serve                 | alias for serve-https"
+	echo "make clean                 | remove all installed dependencies and build artifacts"
+	echo "make deps                  | install all dependencies"
+	echo "make env.json 						 | create an env.json file for the current user's environment"
+	echo "make link                  | create hard links to allow for hot reloading of a built project"
+	echo "make secrets               | symlink secrets files from ../tfvars"
+	echo "make start-with-step       | run the SAM server locally with step function & download lambdas"
+	echo "make style                 | run all style checks"
+	echo "make test                  | run all tests"
+	echo "make cover                 | run all tests with coverage"
+	echo "make env ENV=[env]         | activate env.\$$ENV.json file (default: dev)"
+	echo "make deps-node             | install node dependencies"
+	echo "make deps-python           | install python dependencies"
+	echo "make samconfig.NAME.yaml"  | create a user samconfig file for the specified username"
+	echo "make build								 | build the SAM project for deploying"
+	echo "make deploy                | deploy the SAM project to AWS"
+	echo "make sync                  | sync the SAM project to AWS for quick development"
+	echo "make sync-code             | sync the SAM project to AWS (code changes only)"
+	echo "make serve-http            | run the SAM server locally (HTTP on port 3000)"
+	echo "make serve-https           | run the SAM server locally (HTTPS on port 3002)"
+	echo "make style-node            | run node code style check"
+	echo "make style-python          | run python code style check"
+	echo "make test-node             | run node tests"
+	echo "make test-python           | run python tests"
+	echo "make cover-node            | run node tests with coverage"
+	echo "make cover-python          | run python tests with coverage"
+
+.aws-sam/build.toml: ./template.yaml api/package-lock.json api/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
+	sed -Ei.orig 's/"dependencies"/"devDependencies"/' api/src/package.json
+	cp api/src/package-lock.json api/src/package-lock.json.orig
+	cd api/src && npm i --package-lock-only && cd -
+	for d in . api av-download chat docs ; do \
+		sed -Ei.orig 's/^(\s+)#\*\s/\1/' $$d/template.yaml; \
+	done
+
+	-sam build --cached --parallel
+
+	for d in . api av-download chat docs ; do \
+		mv $$d/template.yaml.orig $$d/template.yaml; \
+	done
+	mv api/src/package.json.orig api/src/package.json
+	mv api/src/package-lock.json.orig api/src/package-lock.json
 layers/ffmpeg/bin/ffmpeg:
-	mkdir -p layers/ffmpeg/bin ;\
+	mkdir -p av-download/layers/ffmpeg/bin ;\
 	curl -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz | \
-	tar -C layers/ffmpeg/bin -xJ --strip-components=1 --wildcards '*/ffmpeg' '*/ffprobe'
-deps-node:
-	cd node/src ;\
+	tar -C av-download/layers/ffmpeg/bin -xJ --strip-components=1 --wildcards '*/ffmpeg' '*/ffprobe'
+deps-api:
+	cd api/src ;\
 	npm list >/dev/null 2>&1 ;\
 	src_deps=$$? ;\
 	cd .. ;\
 	npm list >/dev/null 2>&1 ;\
 	dev_deps=$$? ;\
 	test $$src_deps -eq 0 -a $$dev_deps -eq 0 || npm ci
-
-	cd lambdas ;\
+deps-av-download:
+	cd av-download/lambdas ;\
 	npm list >/dev/null 2>&1 || npm ci
+deps-node: deps-api deps-av-download
 cover-node: deps-node
-	cd node && npm run test:coverage
+	cd api && npm run test:coverage
 style-node: deps-node
-	cd node && npm run prettier
+	cd api && npm run prettier
 test-node: deps-node
-	cd node && npm run test
+	cd api && npm run test
 deps-python:
 	cd chat/src && pip install -r requirements.txt && pip install -r requirements-dev.txt
 cover-python: deps-python
-	cd chat && export SKIP_LLM_REQUEST=True && coverage run --source=src -m unittest -v && coverage report --skip-empty
+	cd chat && coverage run --source=src -m pytest -v && coverage report --skip-empty
 cover-html-python: deps-python
-	cd chat && export SKIP_LLM_REQUEST=True && coverage run --source=src -m unittest -v && coverage html --skip-empty
+	cd chat && coverage run --source=src -m pytest -v && coverage html --skip-empty
 style-python: deps-python
 	cd chat && ruff check . 
 style-python-fix: deps-python
 	cd chat && ruff check --fix . 
 test-python: deps-python
-	cd chat && __SKIP_SECRETS__=true SKIP_LLM_REQUEST=True PYTHONPATH=src:test python -m unittest discover -v
+	cd chat && pytest
 python-version:
 	cd chat && python --version
 build: layers/ffmpeg/bin/ffmpeg .aws-sam/build.toml
+validate:
+	cfn-lint template.yaml **/template.yaml --ignore-checks E3510 W1028 W8001
 serve-http: deps-node
 	@printf '\033[0;31mWARNING: Serving only the local HTTP API. The chat websocket API is not available in local mode.\033[0m\n'
 	rm -rf .aws-sam
-	sam local start-api --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
+	sam local start-api -t api/template.yaml --env-vars $$PWD/env.json --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
 serve-https: SERVE_PARAMS = --port 3002 --ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem
 serve-https: serve-http
 serve: serve-https
-start-with-step: deps-node
-	sam local start-lambda --host 0.0.0.0 --port 3005 --env-vars env.json --log-file lambda.log & \
+start-with-step: deps-node env.json
+	export AWS_DEFAULT_REGION=us-east-1 ;\
+	sam local start-lambda --warm-containers=LAZY -t av-download/template.yaml --host 0.0.0.0 --port 3005 --env-vars $$PWD/env.json --log-file lambda.log & \
 	echo $$! > .sam-pids ;\
-	sam local start-api --host 0.0.0.0 --port 3002 --log-file dc-api.log \
+	sg open all 3005 ;\
+	sam local start-api --warm-containers=LAZY -t api/template.yaml --env-vars $$PWD/env.json --host 0.0.0.0 --port 3002 --log-file dc-api.log \
 		--ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem & \
 	echo $$! >> .sam-pids ;\
 	docker run --rm -p 8083:8083 -e LAMBDA_ENDPOINT=http://172.17.0.1:3005/ amazon/aws-stepfunctions-local ;\
+	echo -n "Shutting down..." ;\
+	sg close all 3005 ;\
 	kill $$(cat .sam-pids) ;\
-	rm -f .sam-pids
+	rm -f .sam-pids ;\
+	echo ""
+state-machine:
+	export TEMPLATE_DIR=$$(mktemp -d); \
+	yq -o=json '.Resources.avDownloadStateMachine.Properties.Definition' av-download/template.yaml > $$TEMPLATE_DIR/av_download.json; \
+	aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://$$TEMPLATE_DIR/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole --no-cli-pager
 deps: deps-node deps-python
 style: style-node style-python
 test: test-node test-python
 cover: cover-node cover-python
-env:
-	ln -fs ./env.${ENV}.json ./env.json
+env.json:
+	./bin/make_env.sh
+samconfig.%.yaml:
+	DEV_PREFIX=$* ./bin/make_deploy_config.sh
+deploy: build samconfig.$(DEV_PREFIX).yaml
+	if ! aws sts get-caller-identity --query 'Arn' --output text | grep AWSReservedSSO_AWSAdministratorAccess > /dev/null; then \
+		echo "You must be logged in as an admin to deploy"; \
+		exit 1; \
+	fi
+	sam deploy --config-file samconfig.$(DEV_PREFIX).yaml --stack-name dc-api-$(DEV_PREFIX)
+sync: samconfig.$(DEV_PREFIX).yaml
+	if ! aws sts get-caller-identity --query 'Arn' --output text | grep AWSReservedSSO_AWSAdministratorAccess > /dev/null; then \
+		echo "You must be logged in as an admin to sync"; \
+		exit 1; \
+	fi
+	sam sync --config-file samconfig.$(DEV_PREFIX).yaml --stack-name dc-api-$(DEV_PREFIX) --watch $(ARGS)
+sync-code: ARGS=--code
+sync-code: sync
 secrets:
-	ln -s ../tfvars/dc-api/* .
+	ln -s ../tfvars/dc-api/*.yaml .
 clean:
-	rm -rf .aws-sam node/node_modules node/src/node_modules python/**/__pycache__ python/.coverage python/.ruff_cache layers/ffmpeg
+	rm -rf .aws-sam api/.aws-sam chat/.aws-sam av-download/.aws-sam api/node_modules api/src/node_modules chat/**/__pycache__ chat/.coverage chat/.ruff_cache
+reset:
+	for f in $$(find . -maxdepth 2 -name '*.orig'); do mv $$f $${f%%.orig}; done