Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions .github/scripts/build-matrix.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/usr/bin/env bash
set -euo pipefail

# Builds a dynamic GitHub Actions matrix for run-samples.yml.
# Usage: build-matrix.sh <run_mode> [base_sha]

# "all" runs every test; "changed" only runs tests whose watch_folders have modified files
RUN_MODE="${1:-all}"
BASE_SHA="${2:-}"

# Get JSON metadata for all tests from run-samples.sh --list
chmod +x ./run-samples.sh
TEST_META=$(./run-samples.sh --list)
TOTAL=$(echo "$TEST_META" | jq length)
echo "Run mode: $RUN_MODE | Total tests: $TOTAL"

# Changes to these files affect all tests, so any modification triggers a full run
INFRA_FILES="run-samples.sh Makefile .github/workflows/run-samples.yml .github/scripts/build-matrix.sh pyproject.toml requirements-dev.txt requirements-runtime.txt"

if [[ "$RUN_MODE" == "changed" && -n "$BASE_SHA" ]]; then
# Get list of files changed between base branch and current HEAD
CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD || true)
echo "Changed files:"
echo "$CHANGED"

# Safety net: if any infrastructure file changed, run all tests
RUN_ALL=false
for f in $INFRA_FILES; do
if echo "$CHANGED" | grep -qF "$f"; then
echo "Infra changed: $f -> running all"
RUN_ALL=true && break
fi
done

if [[ "$RUN_ALL" == "true" ]]; then
INDICES=$(seq 0 $((TOTAL-1)))
else
# Match changed files against each test's watch_folders using prefix matching
INDICES=""
for (( i=0; i<TOTAL; i++ )); do
mapfile -t folders < <(echo "$TEST_META" | jq -r ".[$i].watch_folders[]")
for wf in "${folders[@]}"; do
if echo "$CHANGED" | grep -q "^${wf}/"; then
INDICES+=" $i" && break
fi
done
done
INDICES=$(echo "$INDICES" | xargs)
fi
else
# "all" mode: select every test
INDICES=$(seq 0 $((TOTAL-1)))
fi

# Output the matrix JSON for GitHub Actions
if [[ -z "${INDICES:-}" ]]; then
echo "No tests to run."
echo "has_tests=false" >> "$GITHUB_OUTPUT"
echo 'matrix={"include":[]}' >> "$GITHUB_OUTPUT"
else
echo "has_tests=true" >> "$GITHUB_OUTPUT"
# Convert space-separated indices to JSON array, then build the matrix object
IDX_JSON=$(echo "$INDICES" | tr ' ' '\n' | jq -R 'tonumber' | jq -s '.')
MATRIX=$(echo "$TEST_META" | jq -c --argjson idx "$IDX_JSON" \
'{include: [$idx[] as $i | .[$i] | {shard, splits, name}]}')
echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
echo "Matrix:" && echo "$MATRIX" | jq .
fi
83 changes: 64 additions & 19 deletions .github/workflows/run-samples.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
name: Samples CI

# Theory of Operation:
# This workflow automates the testing of Azure sample applications against the LocalStack Azure emulator.
# It follows the best practices from the localstack-pro repository:
# 1. Parallel Testing: Splits the sample suite into shards to reduce execution time.
# 2. Standardized Tooling: Uses a Makefile for environment setup and test orchestration.
# 3. Cloud Emulation: Configures the Azure CLI to target the LocalStack emulator.
# 4. IaC Coverage: Tests bash scripts, Terraform deployments, and Bicep deployments.
# Tests Azure samples against the LocalStack emulator.
# Each test runs in its own job. Set DEFAULT_RUN_MODE to 'changed' to only run affected tests.

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
Expand All @@ -16,18 +11,68 @@ on:
pull_request:
branches: [ main ]
workflow_dispatch:
inputs:
run_mode:
description: "Test mode: 'all' runs every test, 'changed' runs only tests with modified files"
required: false
type: choice
options:
- all
- changed
default: changed

# Default run mode for pull_request events (change to 'changed' to only run affected tests)
env:
DEFAULT_RUN_MODE: changed

jobs:
# Lightweight job that builds the dynamic test matrix for the main test jobs
setup:
name: "Build Test Matrix"
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.build-matrix.outputs.matrix }}
has_tests: ${{ steps.build-matrix.outputs.has_tests }}
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
fetch-depth: 0 # Full history needed for git diff in "changed" mode

- name: Build dynamic matrix
id: build-matrix
run: |
# Pick run mode: manual dispatch uses the dropdown, PRs use the env default
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
RUN_MODE="${{ github.event.inputs.run_mode }}"
else
RUN_MODE="${{ env.DEFAULT_RUN_MODE }}"
fi

# In "changed" mode, determine the base commit to diff against
BASE_SHA=""
if [[ "$RUN_MODE" == "changed" ]]; then
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
BASE_SHA="${{ github.event.pull_request.base.sha }}"
else
BASE_SHA=$(git rev-parse origin/main 2>/dev/null || git rev-parse HEAD~1)
fi
fi

chmod +x .github/scripts/build-matrix.sh
Comment thread
DrisDary marked this conversation as resolved.
.github/scripts/build-matrix.sh "$RUN_MODE" "$BASE_SHA"

# Each test runs in its own job — one matrix entry per test from the setup job
scripts:
name: "Run Test Scripts (amd64) — Part ${{ matrix.shard }} of ${{ matrix.splits }}"
name: "Test: ${{ matrix.name }}"
needs: setup
if: needs.setup.outputs.has_tests == 'true' # Skip entirely when no tests match
environment: AZURE
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4]
splits: [4]
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
runs-on: github-ubuntu2204-amd64-4

env:
IMAGE_NAME: localstack/localstack-azure-alpha
DEFAULT_TAG: latest
Expand Down Expand Up @@ -67,7 +112,7 @@ jobs:
terraform_wrapper: false

- name: Install test dependencies
# Mirroring the localstack-pro approach: install all Python dependencies
# Mirroring the localstack-pro approach: install all Python dependencies
# (including the localstack CLI) into a virtual environment to avoid system-level conflicts.
run: make install

Expand All @@ -80,14 +125,14 @@ jobs:
password: ${{ secrets.DOCKERHUB_PULL_TOKEN }}

- name: Free up disk space
# Azure emulator images are large. Pruning unused Docker objects ensures enough
# Azure emulator images are large. Pruning unused Docker objects ensures enough
# disk space is available on the GitHub runner for image pulls and sidecar containers.
run: |
docker system prune -af --volumes
docker builder prune -af

- name: Pull LocalStack Azure Image
# Explicitly pull the image before starting. This mirrors the "Build Docker Image"
# Explicitly pull the image before starting. This mirrors the "Build Docker Image"
# step in localstack-pro and ensures the pull logic is separated from the start logic.
run: docker pull ${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}

Expand All @@ -112,7 +157,7 @@ jobs:
run: npm install -g azure-functions-core-tools@4 --unsafe-perm true

- name: Install MSSQL ODBC and Tools
# Required for the 'web-app-sql-database' sample which uses 'sqlcmd' to
# Required for the 'web-app-sql-database' sample which uses 'sqlcmd' to
# initialize and verify the database schema in the local emulator.
run: |
curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc
Expand All @@ -121,9 +166,9 @@ jobs:
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18
echo "/opt/mssql-tools18/bin" >> $GITHUB_PATH

- name: Run Test Scripts
# Executes the sharded test suite. Each shard runs a subset of samples in parallel.
# This includes bash scripts, Terraform deployments, and Bicep deployments.
- name: "Run: ${{ matrix.name }}"
# Each job runs exactly one test. SPLITS equals the total test count, and SHARD
# is the 1-based index of this specific test, so run-samples.sh executes only it.
run: make test SHARD=${{ matrix.shard }} SPLITS=${{ matrix.splits }}
env:
LOCALSTACK_AUTH_TOKEN: ${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }}
Expand Down
104 changes: 65 additions & 39 deletions run-samples.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,71 @@ set -euo pipefail

# 0. Load environment variables from .env file if it exists
if [ -f .env ]; then
echo "Loading environment variables from .env file..."
# Use a subshell to avoid exporting everything if not needed,
echo "Loading environment variables from .env file..." >&2
# Use a subshell to avoid exporting everything if not needed,
# but here we actually want them in the environment.
set -a
source .env
set +a
fi

# 1. Check for required tools
# 1. Define Samples (placed before tool checks so --list works without dependencies)
SAMPLES=(
"samples/function-app-front-door/python|bash scripts/deploy_all.sh --name-prefix testafd --use-localstack|"
"samples/function-app-managed-identity/python|bash scripts/user-managed-identity.sh|bash scripts/validate.sh && bash scripts/test.sh"
"samples/function-app-storage-http/dotnet|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/call-http-triggers.sh"
"samples/web-app-cosmosdb-mongodb-api/python|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/call-web-app.sh"
"samples/web-app-managed-identity/python|bash scripts/user-assigned.sh|bash scripts/validate.sh && bash scripts/call-web-app.sh"
"samples/web-app-sql-database/python|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/get-web-app-url.sh"
)

# 1a. Define Terraform Samples
TERRAFORM_SAMPLES=(
"samples/function-app-managed-identity/python/terraform|bash deploy.sh"
"samples/function-app-storage-http/dotnet/terraform|bash deploy.sh"
"samples/web-app-cosmosdb-mongodb-api/python/terraform|bash deploy.sh"
"samples/web-app-managed-identity/python/terraform|bash deploy.sh"
"samples/web-app-sql-database/python/terraform|bash deploy.sh"
)

# 1b. Define Bicep Samples
BICEP_SAMPLES=(
#"samples/web-app-sql-database/python/bicep|bash deploy.sh"
"samples/function-app-managed-identity/python/bicep|bash deploy.sh"
"samples/function-app-storage-http/dotnet/bicep|bash deploy.sh"
"samples/web-app-cosmosdb-mongodb-api/python/bicep|bash deploy.sh"
"samples/web-app-managed-identity/python/bicep|bash deploy.sh"
)

# Combine script-based, Terraform, and Bicep samples into one array
ALL_SAMPLES=("${SAMPLES[@]}" "${TERRAFORM_SAMPLES[@]}" "${BICEP_SAMPLES[@]}")
TOTAL=${#ALL_SAMPLES[@]}

# 2. Handle --list flag: output JSON metadata for CI matrix generation (no tools required)
# Each entry has: shard (1-based index), splits (total count), name, and watch_folders.
# CI uses watch_folders to detect which tests are affected by changed files.
if [[ "${1:-}" == "--list" ]]; then
echo "["
for (( i=0; i<TOTAL; i++ )); do
IFS='|' read -r path _ _ <<< "${ALL_SAMPLES[$i]}"

if [[ "$path" == */terraform || "$path" == */bicep ]]; then
watch=("$path" "$(dirname "$path")/src")
name="${path#samples/}"
else
watch=("$path/scripts" "$path/src")
name="${path#samples/}/scripts"
fi

printf ' {"shard":%d,"splits":%d,"name":"%s","watch_folders":["%s","%s"]}' \
$((i+1)) "$TOTAL" "$name" "${watch[0]}" "${watch[1]}"
(( i < TOTAL-1 )) && echo "," || echo ""
done
echo "]"
exit 0
fi

# 3. Check for required tools
command -v localstack >/dev/null 2>&1 || { echo >&2 "localstack CLI is required but not installed. Aborting."; exit 1; }
command -v az >/dev/null 2>&1 || { echo >&2 "az CLI is required but not installed. Aborting."; exit 1; }
command -v azlocal >/dev/null 2>&1 || { echo >&2 "azlocal is required but not installed. Run 'pip install azlocal'. Aborting."; exit 1; }
Expand All @@ -41,7 +97,7 @@ if [ -z "${LOCALSTACK_AUTH_TOKEN:-}" ]; then
exit 1
fi

# 1. Start LocalStack
# 4. Start LocalStack
if ! localstack status | grep -q "running"; then
echo "Starting LocalStack Azure emulator..."
IMAGE_NAME=localstack/localstack-azure-alpha localstack start -d
Expand All @@ -50,7 +106,7 @@ else
echo "LocalStack is already running."
fi

# 2. Configure Azure CLI for LocalStack
# 5. Configure Azure CLI for LocalStack
echo "Configuring Azure CLI for LocalStack..."
if [ -n "${AZURE_CONFIG_DIR:-}" ]; then
mkdir -p "$AZURE_CONFIG_DIR"
Expand All @@ -72,53 +128,23 @@ else
az account show --query "{Environment:environmentName, Subscription:id}" --output json 2>&1 || echo "[DEBUG] az account show failed"
fi


# 3. Define Samples
SAMPLES=(
"samples/function-app-front-door/python|bash scripts/deploy_all.sh --name-prefix testafd --use-localstack|"
"samples/function-app-managed-identity/python|bash scripts/user-managed-identity.sh|bash scripts/validate.sh && bash scripts/test.sh"
"samples/function-app-storage-http/dotnet|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/call-http-triggers.sh"
"samples/web-app-cosmosdb-mongodb-api/python|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/call-web-app.sh"
"samples/web-app-managed-identity/python|bash scripts/user-assigned.sh|bash scripts/validate.sh && bash scripts/call-web-app.sh"
"samples/web-app-sql-database/python|bash scripts/deploy.sh|bash scripts/validate.sh && bash scripts/get-web-app-url.sh"
)

# 3a. Define Terraform Samples
TERRAFORM_SAMPLES=(
"samples/function-app-managed-identity/python/terraform|bash deploy.sh"
"samples/function-app-storage-http/dotnet/terraform|bash deploy.sh"
"samples/web-app-cosmosdb-mongodb-api/python/terraform|bash deploy.sh"
"samples/web-app-managed-identity/python/terraform|bash deploy.sh"
"samples/web-app-sql-database/python/terraform|bash deploy.sh"
)

# 3b. Define Bicep Samples
BICEP_SAMPLES=(
#"samples/web-app-sql-database/python/bicep|bash deploy.sh"
"samples/function-app-managed-identity/python/bicep|bash deploy.sh"
"samples/function-app-storage-http/dotnet/bicep|bash deploy.sh"
"samples/web-app-cosmosdb-mongodb-api/python/bicep|bash deploy.sh"
"samples/web-app-managed-identity/python/bicep|bash deploy.sh"
)

# 4. Calculate Shard
# Combine script-based, Terraform, and Bicep samples into one array
ALL_SAMPLES=("${SAMPLES[@]}" "${TERRAFORM_SAMPLES[@]}" "${BICEP_SAMPLES[@]}")
TOTAL=${#ALL_SAMPLES[@]}
# 6. Calculate Shard — determines which slice of ALL_SAMPLES to run.
# When SPLITS=TOTAL, each shard runs exactly 1 test (COUNT=1).
SHARD=${1:-1}
SPLITS=${2:-1}

COUNT=$(( TOTAL / SPLITS ))
START=$(( (SHARD - 1) * COUNT ))

# Last shard picks up any remainder from integer division
if [ "$SHARD" -eq "$SPLITS" ]; then
COUNT=$(( TOTAL - START ))
fi

echo "Running samples shard $SHARD of $SPLITS (index $START, count $COUNT)"
echo "Total samples (scripts + terraform + bicep): $TOTAL"

# 5. Run Samples
# 7. Run Samples — deploy each test, then run its validation if defined
for (( i=START; i<START+COUNT; i++ )); do
item="${ALL_SAMPLES[$i]}"
IFS='|' read -r path deploy test <<< "$item"
Expand Down