Download from S3 if not found on EFS #25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Main build pipeline that verifies, builds, and deploys the software | |
| name: Build and Deploy | |
| # Events that trigger the workflow | |
| on: | |
| # Trigger based on push to all branches - TODO | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| - 'release/**' | |
| - 'feature/**' | |
| - 'issue/**' | |
| - 'issues/**' | |
| - 'dependabot/**' | |
| tags-ignore: | |
| - '*' | |
| # Do not trigger build if pyproject.toml was the only thing changed | |
| paths-ignore: | |
| - 'pyproject.toml' | |
| - 'poetry.lock' | |
| # Run workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| venue: | |
| type: choice | |
| description: Venue to deploy to | |
| options: | |
| - SIT | |
| - UAT | |
| - OPS | |
| # Only allow 1 execution of this workflow to be running at any given time per-branch. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| # Environment variables | |
| env: | |
| APP_NAME_ENV: 'publish-cnm' | |
| POETRY_VERSION: "2.1.1" | |
| PYTHON_VERSION: "3.10" | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| build: | |
| name: Build, test, verify | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| outputs: | |
| app_version: ${{ steps.version.outputs.APP_VERSION }} | |
| prefix_env: ${{ steps.version.outputs.PREFIX_ENV }} | |
| target_env: ${{ steps.version.outputs.TARGET_ENV }} | |
| target_env_lowercase: ${{ steps.version.outputs.TARGET_ENV_LOWERCASE }} | |
| steps: | |
| # Create a release and set deployment environment | |
| - uses: getsentry/action-github-app-token@v3 | |
| name: PO.DAAC CICD token | |
| id: podaac-cicd | |
| with: | |
| app_id: ${{ secrets.CICD_APP_ID }} | |
| private_key: ${{ secrets.CICD_APP_PRIVATE_KEY }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository }} | |
| token: ${{ steps.podaac-cicd.outputs.token }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install poetry | |
| uses: abatilo/actions-poetry@v3 | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Setup a local virtual environment | |
| run: | | |
| poetry config virtualenvs.create true --local | |
| poetry config virtualenvs.in-project true --local | |
| # Get current version | |
| - name: Get pre-build version | |
| id: get-version | |
| run: | | |
| echo "poetry version" | |
| echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT | |
| echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV | |
| - name: Manual build | |
| # If triggered by workflow dispatch, no version bump | |
| if: github.event_name == 'workflow_dispatch' | |
| id: manual | |
| run: | | |
| echo "APP_VERSION=${{ steps.get-version.outputs.current_version }}" >> $GITHUB_ENV | |
| echo "TARGET_ENV=${{ github.event.inputs.venue }}" >> $GITHUB_ENV | |
| target_env=${{ github.event.inputs.venue }} | |
| echo "PREFIX_ENV=svc-confluence-$(echo "$target_env" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV | |
| - name: Bump pre-alpha version | |
| # If triggered by push to a non-tracked branch (DEV1) | |
| if: | | |
| github.ref != 'refs/heads/develop' && | |
| github.ref != 'refs/heads/main' && | |
| !startsWith(github.ref, 'refs/heads/release/') | |
| run: | | |
| new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})" | |
| poetry version $new_ver | |
| echo "APP_VERSION=$new_ver" >> $GITHUB_ENV | |
| echo "TARGET_ENV=SIT" >> $GITHUB_ENV | |
| echo "PREFIX_ENV=svc-confluence-sit" >> $GITHUB_ENV | |
| - name: Bump alpha version | |
| # If triggered by push to the develop branch (DEV1) | |
| if: | | |
| github.ref == 'refs/heads/develop' && | |
| steps.manual.conclusion == 'skipped' | |
| id: alpha | |
| run: | | |
| poetry version prerelease | |
| echo "APP_VERSION=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV | |
| echo "TARGET_ENV=SIT" >> $GITHUB_ENV | |
| echo "PREFIX_ENV=svc-confluence-sit" >> $GITHUB_ENV | |
| - name: Bump rc version | |
| # If triggered by push to a release branch (DEV2) | |
| if: | | |
| startsWith(github.ref, 'refs/heads/release/') && | |
| steps.manual.conclusion == 'skipped' | |
| id: rc | |
| env: | |
| # True if the version already has a 'rc' pre-release identifier | |
| BUMP_RC: ${{ contains(steps.get-version.outputs.current_version, 'rc') }} | |
| run: | | |
| if [ "$BUMP_RC" = true ]; then | |
| poetry version prerelease | |
| else | |
| poetry version ${GITHUB_REF#refs/heads/release/}rc1 | |
| fi | |
| echo "APP_VERSION=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV | |
| echo "TARGET_ENV=UAT" >> $GITHUB_ENV | |
| echo "PREFIX_ENV=svc-confluence-uat" >> $GITHUB_ENV | |
| - name: Release version | |
| # If triggered by push to the main branch (OPS) | |
| if: | | |
| startsWith(github.ref, 'refs/heads/main') && | |
| steps.manual.conclusion == 'skipped' | |
| id: release | |
| env: | |
| CURRENT_VERSION: ${{ steps.get-version.outputs.current_version }} | |
| # Remove rc* from end of version string | |
| # The ${string%%substring} syntax below deletes the longest match of $substring from back of $string. | |
| run: | | |
| poetry version ${CURRENT_VERSION%%rc*} | |
| echo "APP_VERSION=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV | |
| echo "TARGET_ENV=OPS" >> $GITHUB_ENV | |
| echo "PREFIX_ENV=svc-confluence-ops" >> $GITHUB_ENV | |
| # Validate infrastructure terraform files | |
| - name: Setup terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| - name: Validate terraform | |
| working-directory: terraform/ | |
| run: | | |
| terraform init -backend=false | |
| terraform validate -no-color | |
| # SNYK IAC scan and report | |
| - name: Run Snyk IAC to test and report | |
| uses: snyk/actions/iac@master | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| command: test | |
| file: terraform/ | |
| args: > | |
| --org=${{ secrets.SNYK_ORG_ID }} | |
| --severity-threshold=high | |
| --report | |
| # SNYK Python | |
| - name: Run Snyk Python to test | |
| uses: snyk/actions/python-3.10@master | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| command: test | |
| args: > | |
| --org=${{ secrets.SNYK_ORG_ID }} | |
| --project-name=${{ github.repository }} | |
| --severity-threshold=high | |
| --fail-on=all | |
| - name: Run Snyk Python to report | |
| uses: snyk/actions/python-3.10@master | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| command: monitor | |
| args: > | |
| --org=${{ secrets.SNYK_ORG_ID }} | |
| --project-name=${{ github.repository }} | |
| - name: Commit version bump | |
| # If building an alpha, release candidate, or release then we commit the version bump back to the repo | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| run: | | |
| git config user.name "${GITHUB_ACTOR}" | |
| git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git commit -am "/version ${{ env.APP_VERSION }}" | |
| git push | |
| - name: Push tag | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| run: | | |
| git config user.name "${GITHUB_ACTOR}" | |
| git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git tag -a "${{ env.APP_VERSION }}" -m "Version ${{ env.APP_VERSION }}" | |
| git push origin "${{ env.APP_VERSION }}" | |
| - name: Create GH release | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| generateReleaseNotes: true | |
| name: ${{ env.APP_VERSION }} | |
| prerelease: ${{ steps.alpha.conclusion == 'success' || steps.rc.conclusion == 'success'}} | |
| tag: ${{ env.APP_VERSION }} | |
| - name: Save application version to output | |
| id: version | |
| run: | | |
| echo "APP_VERSION=${{ env.APP_VERSION }}" >> $GITHUB_OUTPUT | |
| echo "PREFIX_ENV=${{ env.PREFIX_ENV }}" >> $GITHUB_OUTPUT | |
| echo "TARGET_ENV=${{ env.TARGET_ENV }}" >> $GITHUB_OUTPUT | |
| echo TARGET_ENV_LOWERCASE=${{ env.TARGET_ENV }} | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT" | |
| container: | |
| name: Build container image | |
| # The type of runner that the job will run on | |
| runs-on: ubuntu-latest | |
| needs: build | |
| env: | |
| APP_VERSION: ${{ needs.build.outputs.app_version }} | |
| PREFIX_ENV: ${{ needs.build.outputs.prefix_env }} | |
| TARGET_ENV: ${{ needs.build.outputs.target_env }} | |
| steps: | |
| # Check out GitHub repo | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository }} | |
| # ghcr.io container image | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=pep440,pattern={{version}},value=${{ env.APP_VERSION }} | |
| type=raw,value=${{ env.TARGET_ENV }} | |
| - name: Build and push Docker image | |
| if: | | |
| github.ref == 'refs/heads/develop' || | |
| github.ref == 'refs/heads/main' || | |
| startsWith(github.ref, 'refs/heads/release') || | |
| github.event_name == 'workflow_dispatch' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| push: true | |
| pull: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| # AWS ECR repository | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV)] }} | |
| aws-secret-access-key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV)] }} | |
| aws-region: us-west-2 | |
| mask-aws-account-id: true | |
| - name: Login to AWS ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| mask-password: 'true' | |
| - name: Define ECR registry, repository, and image tag names | |
| run : | | |
| echo "REGISTRY=${{ steps.login-ecr.outputs.registry }}" >> $GITHUB_ENV | |
| echo "REPOSITORY=${{ env.PREFIX_ENV }}-${{ env.APP_NAME_ENV }}" >> $GITHUB_ENV | |
| image_repository=${{ fromJSON(steps.meta.outputs.json).tags[0] }} | |
| image_tag=$(echo ${image_repository##*:}) | |
| echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV | |
| # Create ECR repository (if it does not exist) | |
| - name: Create AWS ECR Repository | |
| run: deploy/deploy-ecr.sh $REGISTRY $REPOSITORY | |
| # Build and push Docker container image by version tag and to latest tag | |
| - name: Build and Push to AWS ECR | |
| run: | | |
| docker tag ${{ fromJSON(steps.meta.outputs.json).tags[0] }} $REGISTRY/$REPOSITORY:$IMAGE_TAG | |
| docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG | |
| docker tag ${{ fromJSON(steps.meta.outputs.json).tags[0] }} $REGISTRY/$REPOSITORY:latest | |
| docker push $REGISTRY/$REPOSITORY:latest | |
| deploy: | |
| name: Deploy module | |
| # The type of runner that the job will run on | |
| runs-on: ubuntu-latest | |
| needs: [build, container] | |
| env: | |
| APP_VERSION: ${{ needs.build.outputs.app_version }} | |
| PREFIX_ENV: ${{ needs.build.outputs.prefix_env }} | |
| TARGET_ENV: ${{ needs.build.outputs.target_env }} | |
| TARGET_ENV_LOWERCASE: ${{ needs.build.outputs.target_env_lowercase }} | |
| steps: | |
| # Check out GitHub repo | |
| - uses: actions/checkout@v4 | |
| # Configure credentials | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV)] }} | |
| aws-secret-access-key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV)] }} | |
| aws-region: us-west-2 | |
| mask-aws-account-id: true | |
| # Set up Terraform | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| - name: Define TF_VAR values | |
| run: | | |
| echo "TF_VAR_app_version=${{ env.APP_VERSION }}" >> $GITHUB_ENV | |
| echo "TF_VAR_collection=${{ secrets['COLLECTION'] }}" >> $GITHUB_ENV | |
| echo "TF_VAR_cross_account=${{ secrets[format('CROSS_ACCOUNT_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV | |
| echo "TF_VAR_data_provider=${{ secrets['PROVIDER'] }}" >> $GITHUB_ENV | |
| echo "TF_VAR_environment=${{ env.TARGET_ENV }}" >> $GITHUB_ENV | |
| echo "TF_VAR_podaac_key=${{ secrets['PODAAC_KEY'] }}" >> $GITHUB_ENV | |
| echo "TF_VAR_podaac_secret=${{ secrets['PODAAC_SECRET'] }}" >> $GITHUB_ENV | |
| echo "TF_VAR_prefix=${{ env.PREFIX_ENV }}" >> $GITHUB_ENV | |
| echo "TF_VAR_provider_version=${{ secrets['VERSION'] }}" >> $GITHUB_ENV | |
| echo "TF_VAR_sns_topic_arn=${{ secrets[format('SNS_TOPIC_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV | |
| echo "TF_IN_AUTOMATION=true" >> $GITHUB_ENV | |
| - name: Initialize Terraform | |
| working-directory: terraform/ | |
| run: | | |
| terraform init -reconfigure \ | |
| -backend-config="bucket=podaac-services-${{ env.TARGET_ENV_LOWERCASE }}-terraform" \ | |
| -backend-config="key=services/confluence/${{ env.APP_NAME_ENV }}.tfstate" \ | |
| -backend-config="region=${AWS_DEFAULT_REGION}" | |
| # Deploy AWS infrastructure | |
| - name: Deploy Terraform | |
| working-directory: terraform/ | |
| run: terraform apply -auto-approve |