Accelerated Python/Kernels: Update colab install instructions in solu… #130
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Push Brev Tutorial Docker Images | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - "pull-request/[0-9]+" | |
| workflow_dispatch: | |
| jobs: | |
| discover-tutorials: | |
| runs-on: linux-amd64-cpu4 | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| outputs: | |
| tutorials: ${{ steps.find-tutorials.outputs.tutorials }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Find tutorial directories | |
| id: find-tutorials | |
| run: | | |
| tutorials=$(brev/discover-tutorials.bash | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "tutorials=${tutorials}" >> $GITHUB_OUTPUT | |
| echo "Found tutorials: ${tutorials}" | |
| build-and-push: | |
| needs: discover-tutorials | |
| runs-on: linux-amd64-cpu4 | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }} | |
| permissions: | |
| contents: write | |
| packages: write | |
| actions: write | |
| statuses: write | |
| strategy: | |
| matrix: | |
| tutorial: ${{ fromJson(needs.discover-tutorials.outputs.tutorials) }} | |
| fail-fast: false | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set Git branch variables | |
| run: | | |
| GIT_BRANCH_NAME=${GITHUB_REF#refs/heads/} | |
| # Sanitize branch name for Docker tags (replace invalid characters with hyphens and convert to lowercase) | |
| DOCKER_TAG_BRANCH=$(echo "${GIT_BRANCH_NAME}" | sed 's/[^a-zA-Z0-9._-]/-/g' | tr '[:upper:]' '[:lower:]') | |
| GIT_SHA=${{ github.sha }} | |
| GIT_SHORT_SHA=${GIT_SHA::7} | |
| echo "GIT_BRANCH_NAME=${GIT_BRANCH_NAME}" >> $GITHUB_ENV | |
| echo "DOCKER_TAG_BRANCH=${DOCKER_TAG_BRANCH}" >> $GITHUB_ENV | |
| echo "GIT_SHA=${GIT_SHA}" >> $GITHUB_ENV | |
| echo "GIT_SHORT_SHA=${GIT_SHORT_SHA}" >> $GITHUB_ENV | |
| - name: Set image name and tutorial name | |
| run: | | |
| TUTORIAL_NAME=$(basename ${{ matrix.tutorial }}) | |
| IMAGE_NAME="ghcr.io/${{ github.repository_owner }}/${TUTORIAL_NAME}-tutorial" | |
| echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV | |
| echo "TUTORIAL_NAME=${TUTORIAL_NAME}" >> $GITHUB_ENV | |
| - name: Check for HPCCM recipe | |
| run: | | |
| if [ -f "${{ matrix.tutorial }}/brev/docker-recipe.py" ]; then | |
| HAS_HPCCM=true | |
| else | |
| HAS_HPCCM=false | |
| fi | |
| echo "HAS_HPCCM=${HAS_HPCCM}" >> $GITHUB_ENV | |
| - name: Set up Python (for HPCCM) | |
| if: env.HAS_HPCCM == 'true' | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install HPCCM | |
| if: env.HAS_HPCCM == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install hpccm | |
| - name: Generate Dockerfile from HPCCM recipe | |
| if: env.HAS_HPCCM == 'true' | |
| run: | | |
| hpccm --recipe ${{ matrix.tutorial }}/brev/docker-recipe.py --format docker > ${{ matrix.tutorial }}/brev/dockerfile | |
| - name: Generate Docker Compose files | |
| run: | | |
| python3 brev/generate-tagged-docker-composes.py \ | |
| --image-tag "${DOCKER_TAG_BRANCH}-git-${GIT_SHORT_SHA}" \ | |
| --output-dir "artifacts/commit-specific" \ | |
| --tutorial "${TUTORIAL_NAME}" \ | |
| --type all | |
| python3 brev/generate-tagged-docker-composes.py \ | |
| --image-tag "${DOCKER_TAG_BRANCH}-latest" \ | |
| --output-dir "artifacts/branch-latest" \ | |
| --tutorial "${TUTORIAL_NAME}" \ | |
| --type all | |
| - name: Upload commit-specific Docker Compose artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-compose-${{ env.TUTORIAL_NAME }}-${{ env.DOCKER_TAG_BRANCH }}-git-${{ env.GIT_SHORT_SHA }} | |
| path: artifacts/commit-specific/${{ env.TUTORIAL_NAME }}/ | |
| retention-days: 30 | |
| - name: Push branch-latest Docker Compose to generated branch | |
| run: | | |
| # Configure git | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| # Fetch the generated branch | |
| git fetch origin generated:generated | |
| git checkout generated | |
| # Create directory structure for this branch and tutorial | |
| mkdir -p "${GIT_BRANCH_NAME}/tutorials" | |
| # Remove only this tutorial's old files (if they exist) | |
| rm -rf "${GIT_BRANCH_NAME}/tutorials/${TUTORIAL_NAME}" | |
| # Copy this tutorial's generated files | |
| cp -r artifacts/branch-latest/${TUTORIAL_NAME} "${GIT_BRANCH_NAME}/tutorials/" | |
| # Stage only this tutorial's directory | |
| git add "${GIT_BRANCH_NAME}/tutorials/${TUTORIAL_NAME}" | |
| # Check if there are changes to commit | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit for ${TUTORIAL_NAME}" | |
| else | |
| git commit -m "Update docker-compose for ${GIT_BRANCH_NAME}/${TUTORIAL_NAME}" | |
| # Push with retry logic for concurrent updates | |
| for i in {1..5}; do | |
| if git push origin generated; then | |
| echo "✓ Successfully pushed to generated branch" | |
| break | |
| else | |
| echo "Push failed (attempt $i/5), pulling and retrying..." | |
| git pull --rebase origin generated | |
| sleep 2 | |
| fi | |
| done | |
| fi | |
| # Return to original branch | |
| git checkout ${GITHUB_REF#refs/heads/} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push Docker image (with retry) | |
| uses: nick-fields/retry@v3 | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| with: | |
| timeout_minutes: 60 | |
| max_attempts: 3 | |
| retry_wait_seconds: 30 | |
| command: | | |
| cd ${{ matrix.tutorial }}/brev | |
| docker buildx bake \ | |
| --allow=fs.read=/home/runner \ | |
| --set "base.output=type=registry" \ | |
| --set "base.tags=${IMAGE_NAME}:${DOCKER_TAG_BRANCH}-latest" \ | |
| --set "base.tags=${IMAGE_NAME}:${DOCKER_TAG_BRANCH}-git-${GIT_SHORT_SHA}" \ | |
| $([ "${GIT_BRANCH_NAME}" = "main" ] && echo "--set base.tags=${IMAGE_NAME}:latest") \ | |
| --set "base.cache-from=type=registry,ref=${IMAGE_NAME}:buildcache-${DOCKER_TAG_BRANCH},oci-mediatypes=true" \ | |
| --set "base.cache-from=type=registry,ref=${IMAGE_NAME}:buildcache-main,oci-mediatypes=true" \ | |
| --set "base.cache-to=type=registry,ref=${IMAGE_NAME}:buildcache-${DOCKER_TAG_BRANCH},mode=max,oci-mediatypes=true,compression=zstd,compression-level=3" \ | |
| --set "base.platform=linux/amd64" \ | |
| -f docker-compose.yml \ | |
| base | |
| - name: Create pending commit status for test | |
| run: | | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| /repos/${{ github.repository }}/statuses/${GIT_SHA} \ | |
| -f state='pending' \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/workflows/test-brev-tutorial-docker-images.yml" \ | |
| -f description='Test started' \ | |
| -f context="test / ${TUTORIAL_NAME}" | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Trigger test workflow | |
| run: | | |
| echo "Triggering test for tutorial: ${TUTORIAL_NAME}" | |
| gh workflow run test-brev-tutorial-docker-images.yml \ | |
| --ref ${{ github.ref_name }} \ | |
| -f tutorial=${TUTORIAL_NAME} \ | |
| -f git_sha=${GIT_SHA} \ | |
| -f workflow_run_id=${{ github.run_id }} | |
| echo "Test workflow triggered successfully" | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Check disk space after build | |
| if: always() | |
| run: | | |
| echo "Disk space after build:" | |
| df -h / | |
| echo "" | |
| echo "Disk usage by directory:" | |
| du -sh /var/lib/docker/* 2>/dev/null || true | |
| du -sh /var/lib/buildkit/* 2>/dev/null || true |