Skip to content

Commit 26ecd68

Browse files
aofeiaofei
authored
feat(spx-backend): implement Database Migration System (#2001)
Fixes #1958 Signed-off-by: Aofei Sheng <[email protected]>
1 parent 77f23b4 commit 26ecd68

File tree

16 files changed

+917
-94
lines changed

16 files changed

+917
-94
lines changed

‎.github/workflows/validate-database-migrations.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: Validate database migrations
2+
on:
3+
push:
4+
paths:
5+
- spx-backend/internal/migration/migrations/**
6+
- scripts/validate-database-migrations.sh
7+
pull_request:
8+
paths:
9+
- spx-backend/internal/migration/migrations/**
10+
- scripts/validate-database-migrations.sh
11+
jobs:
12+
validate-database-migrations:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Check out code
16+
uses: actions/checkout@v5
17+
- name: Run database migration validation
18+
run: ./scripts/validate-database-migrations.sh

‎.github/workflows/validate.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,7 @@ name: Validate (lint, test & ...)
22

33
on:
44
push:
5-
branches:
6-
- '**'
75
pull_request:
8-
branches:
9-
- '**'
106

117
jobs:
128
spx-gui-lint:

‎scripts/validate-database-migrations.sh‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
# Environment variable configuration
6+
MYSQL_VERSION="${MYSQL_VERSION:-5.7}"
7+
MIGRATE_VERSION="${MIGRATE_VERSION:-4}"
8+
9+
# Internal configuration
10+
MYSQL_CONTAINER_NAME="xbuilder-database-migration-test-$(date +%s)-$$"
11+
MYSQL_PORT=$(shuf -i 13306-19999 -n 1)
12+
MYSQL_HOST="127.0.0.1"
13+
MYSQL_USER="root"
14+
MYSQL_PASSWORD="root"
15+
MYSQL_DATABASE="xbuilder"
16+
MIGRATIONS_TABLE_NAME="schema_migration"
17+
MIGRATIONS_DIR="spx-backend/internal/migration/migrations"
18+
TMPDIR="$(mktemp -d)"
19+
export TMPDIR
20+
SNAPSHOTS_DIR=$(mktemp -d)
21+
22+
# Logging functions
23+
log_error() {
24+
echo "[ERROR] $1"
25+
}
26+
log_error_and_exit() {
27+
log_error "$1"
28+
exit 1
29+
}
30+
log_info() {
31+
echo "[INFO] $1"
32+
}
33+
log_ok() {
34+
echo "[OK] $1"
35+
}
36+
37+
# Set up cleanup trap
38+
cleanup() {
39+
log_info "Cleaning up..."
40+
41+
# Remove temporary container
42+
if docker ps -q -f name="${MYSQL_CONTAINER_NAME}" | grep -q .; then
43+
log_info "Stopping MySQL container: ${MYSQL_CONTAINER_NAME}"
44+
docker rm -f "${MYSQL_CONTAINER_NAME}" &>/dev/null || true
45+
fi
46+
47+
# Remove temporary directory
48+
rm -rf "${TMPDIR}"
49+
}
50+
trap cleanup EXIT INT TERM
51+
52+
# Display initial information
53+
log_info "MySQL version: ${MYSQL_VERSION}"
54+
log_info "Migrate version: ${MIGRATE_VERSION}"
55+
56+
echo
57+
echo "========================================================================="
58+
echo
59+
60+
# Check if Docker is available
61+
command -v docker &> /dev/null || log_error_and_exit "Docker not found. Please install Docker and try again."
62+
63+
# Pull necessary Docker images
64+
log_info "Pulling necessary Docker images..."
65+
docker pull --platform linux/amd64 mysql:${MYSQL_VERSION} >/dev/null || log_error_and_exit "Failed to pull MySQL Docker image"
66+
docker pull --platform linux/amd64 migrate/migrate:${MIGRATE_VERSION} >/dev/null || log_error_and_exit "Failed to pull migrate Docker image"
67+
68+
echo
69+
echo "========================================================================="
70+
echo
71+
72+
# Start new MySQL container
73+
log_info "Creating temporary MySQL container (port: ${MYSQL_PORT})"
74+
docker run \
75+
-d \
76+
--platform linux/amd64 \
77+
--name "${MYSQL_CONTAINER_NAME}" \
78+
-e MYSQL_ROOT_PASSWORD="${MYSQL_PASSWORD}" \
79+
-e MYSQL_DATABASE="${MYSQL_DATABASE}" \
80+
-p "${MYSQL_PORT}:3306" \
81+
mysql:${MYSQL_VERSION} >/dev/null || log_error_and_exit "Failed to start MySQL container"
82+
log_ok "Container started: ${MYSQL_CONTAINER_NAME}"
83+
84+
# Construct MySQL DSN
85+
MYSQL_DSN="mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_HOST}:${MYSQL_PORT})/${MYSQL_DATABASE}?charset=utf8mb4&parseTime=True&loc=UTC&multiStatements=true&x-migrations-table=${MIGRATIONS_TABLE_NAME}"
86+
87+
# MySQL command runner
88+
run_mysql_command() {
89+
docker run --rm --platform linux/amd64 --network host mysql:${MYSQL_VERSION} \
90+
"$@" \
91+
-h "${MYSQL_HOST}" -P "${MYSQL_PORT}" -u "${MYSQL_USER}" -p"${MYSQL_PASSWORD}"
92+
}
93+
94+
log_info "Waiting for MySQL to be ready..."
95+
for i in {1..60}; do
96+
if run_mysql_command mysqladmin ping --silent 2>/dev/null; then
97+
log_ok "MySQL is ready"
98+
break
99+
fi
100+
101+
if [[ $i -eq 60 ]]; then
102+
docker logs "${MYSQL_CONTAINER_NAME}" 2>/dev/null || true
103+
log_error_and_exit "MySQL failed to start within 120 seconds"
104+
fi
105+
106+
if [[ $((i % 10)) -eq 0 ]]; then
107+
log_info "Still waiting... (${i}/60)"
108+
fi
109+
110+
sleep 2
111+
done
112+
113+
echo
114+
echo "========================================================================="
115+
echo
116+
117+
# Function to dump database schema
118+
dump_schema() {
119+
local output_file="$1"
120+
local description="$2"
121+
122+
log_info "Creating schema snapshot: ${description}"
123+
run_mysql_command mysqldump \
124+
--no-data \
125+
--skip-comments \
126+
--skip-add-locks \
127+
--skip-add-drop-table \
128+
--compact \
129+
--single-transaction \
130+
--ignore-table="${MYSQL_DATABASE}.${MIGRATIONS_TABLE_NAME}" \
131+
"${MYSQL_DATABASE}" > "${output_file}" 2>/dev/null || {
132+
# Handle empty database case
133+
touch "${output_file}"
134+
}
135+
136+
# Normalize schema for comparison
137+
# Remove AUTO_INCREMENT values and sort for consistent comparison
138+
if [[ -s "${output_file}" ]]; then
139+
sed -i.bak 's/AUTO_INCREMENT=[0-9]*//' "${output_file}" && rm "${output_file}.bak"
140+
sort "${output_file}" -o "${output_file}"
141+
fi
142+
}
143+
144+
# Function to check if database is empty (excluding migration tracking table)
145+
is_database_empty() {
146+
local table_count
147+
table_count=$(run_mysql_command mysql -e "SELECT COUNT(*) AS count FROM information_schema.tables WHERE table_schema='${MYSQL_DATABASE}' AND table_name != '${MIGRATIONS_TABLE_NAME}'" -s -N 2>/dev/null || echo "0")
148+
[[ "${table_count}" -eq 0 ]]
149+
}
150+
151+
# Function to get migration version from filename
152+
get_version() {
153+
local filename="$1"
154+
echo "${filename}" | grep -oE '^[0-9]+' || echo "000"
155+
}
156+
157+
# Function to compare schemas
158+
compare_schemas() {
159+
local expected_file="$1"
160+
local actual_file="$2"
161+
local version="$3"
162+
163+
if ! diff -u "${expected_file}" "${actual_file}" > "${SNAPSHOTS_DIR}/diff_${version}.txt"; then
164+
log_error "Schema mismatch detected at version ${version}"
165+
log_info "Expected schema: ${expected_file}"
166+
log_info "Actual schema: ${actual_file}"
167+
log_info "Diff saved to: ${SNAPSHOTS_DIR}/diff_${version}.txt"
168+
echo
169+
echo "Differences:"
170+
head -20 "${SNAPSHOTS_DIR}/diff_${version}.txt"
171+
return 1
172+
else
173+
log_ok "Schema matches perfectly for version ${version}"
174+
return 0
175+
fi
176+
}
177+
178+
log_info "Phase 1: Validating UP migrations"
179+
180+
# Verify we start with an empty database
181+
is_database_empty || log_error_and_exit "Database is not empty at start"
182+
183+
# Create initial empty schema snapshot
184+
dump_schema "${SNAPSHOTS_DIR}/schema_000.sql" "Initial empty database"
185+
log_ok "Confirmed database is empty"
186+
187+
# Get all UP migration files and sort by version
188+
up_migrations=($(find "${MIGRATIONS_DIR}" -name "*.up.sql" | sort))
189+
190+
[[ ${#up_migrations[@]} -eq 0 ]] && log_error_and_exit "No UP migration files found in ${MIGRATIONS_DIR}"
191+
192+
log_info "Found ${#up_migrations[@]} UP migration files"
193+
194+
# Execute UP migrations
195+
for migration_file in "${up_migrations[@]}"; do
196+
filename=$(basename "${migration_file}")
197+
version=$(get_version "${filename}")
198+
199+
log_info "Executing UP migration: ${filename}"
200+
201+
# Execute migration using Docker
202+
if docker run --rm -v "$(pwd)/${MIGRATIONS_DIR}:/migrations" --network host migrate/migrate:${MIGRATE_VERSION} -path /migrations -database "${MYSQL_DSN}" up 1; then
203+
log_ok "UP migration ${filename} executed successfully"
204+
else
205+
log_error_and_exit "UP migration ${filename} failed"
206+
fi
207+
208+
# Create schema snapshot
209+
dump_schema "${SNAPSHOTS_DIR}/schema_${version}.sql" "After UP migration ${filename}"
210+
done
211+
212+
log_ok "All UP migrations executed successfully"
213+
214+
echo
215+
echo "========================================================================="
216+
echo
217+
218+
log_info "Phase 2: Validating DOWN migrations"
219+
220+
# Get all DOWN migration files and sort by version (descending)
221+
down_migrations=($(find "${MIGRATIONS_DIR}" -name "*.down.sql" | sort -r))
222+
223+
[[ ${#down_migrations[@]} -eq 0 ]] && log_error_and_exit "No DOWN migration files found in ${MIGRATIONS_DIR}"
224+
225+
log_info "Found ${#down_migrations[@]} DOWN migration files"
226+
227+
# Execute DOWN migrations and compare schemas
228+
for migration_file in "${down_migrations[@]}"; do
229+
filename=$(basename "${migration_file}")
230+
version=$(get_version "${filename}")
231+
232+
# Calculate expected version (previous version)
233+
expected_version=$(printf "%03d" $((10#${version} - 1)))
234+
235+
log_info "Executing DOWN migration: ${filename}"
236+
237+
# Execute migration using Docker
238+
if docker run --rm -v "$(pwd)/${MIGRATIONS_DIR}:/migrations" --network host migrate/migrate:${MIGRATE_VERSION} -path /migrations -database "${MYSQL_DSN}" down 1; then
239+
log_ok "DOWN migration ${filename} executed successfully"
240+
else
241+
log_error_and_exit "DOWN migration ${filename} failed"
242+
fi
243+
244+
# Create current schema snapshot
245+
dump_schema "${SNAPSHOTS_DIR}/schema_after_down_${version}.sql" "After DOWN migration ${filename}"
246+
247+
# Compare with expected schema
248+
expected_schema="${SNAPSHOTS_DIR}/schema_${expected_version}.sql"
249+
actual_schema="${SNAPSHOTS_DIR}/schema_after_down_${version}.sql"
250+
251+
if [[ -f "${expected_schema}" ]]; then
252+
if compare_schemas "${expected_schema}" "${actual_schema}" "${version}"; then
253+
log_ok "Schema symmetry verified for migration ${version}"
254+
else
255+
log_error_and_exit "Schema symmetry FAILED for migration ${version}"
256+
fi
257+
else
258+
log_error_and_exit "Expected schema file not found: ${expected_schema}"
259+
fi
260+
done
261+
262+
# Final check: ensure database is empty
263+
if is_database_empty; then
264+
log_ok "Database is empty after all DOWN migrations"
265+
else
266+
echo "Remaining tables:"
267+
run_mysql_command mysql -e "SELECT table_name FROM information_schema.tables WHERE table_schema='${MYSQL_DATABASE}' AND table_name != '${MIGRATIONS_TABLE_NAME}'" -s -N "${MYSQL_DATABASE}" 2>/dev/null || true
268+
log_error_and_exit "Database is not empty after all DOWN migrations"
269+
fi
270+
271+
echo
272+
echo "========================================================================="
273+
echo
274+
275+
log_ok "All migration validation tests passed"
276+
log_ok "UP migrations execute without errors"
277+
log_ok "DOWN migrations execute without errors"
278+
log_ok "Migration symmetry is perfect"
279+
log_ok "Database returns to empty state"
280+
281+
echo
282+
echo "========================================================================="
283+
echo

‎spx-backend/.env.dev‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
PORT=:8080
22
ALLOWED_ORIGIN=*
33
# Use local DB by default for dev
4-
GOP_SPX_DSN=root:123456@tcp(127.0.0.1:3306)/builder?charset=utf8mb4&parseTime=True&loc=UTC
4+
GOP_SPX_DSN=root:123456@tcp(127.0.0.1:3306)/builder?charset=utf8mb4&parseTime=True&loc=UTC&multiStatements=true
5+
GOP_SPX_AUTO_MIGRATE=true
6+
GOP_SPX_MIGRATION_TIMEOUT=10m
57
# AIGC Service
68
AIGC_ENDPOINT=http://36.213.14.15:8888
79

‎spx-backend/cmd/spx-backend/main.yap‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/goplus/builder/spx-backend/internal/config"
1818
"github.com/goplus/builder/spx-backend/internal/controller"
1919
"github.com/goplus/builder/spx-backend/internal/log"
20+
"github.com/goplus/builder/spx-backend/internal/migration"
2021
"github.com/goplus/builder/spx-backend/internal/model"
2122
)
2223

@@ -48,6 +49,14 @@ if err != nil {
4849
}
4950
defer sentry.Flush(10 * time.Second)
5051

52+
// Execute automatic migration for database if enabled.
53+
if cfg.Database.AutoMigrate {
54+
migrator := migration.New(cfg.Database.DSN, cfg.Database.GetMigrationTimeout())
55+
if err := migrator.Migrate(); err != nil {
56+
logger.Fatalln("failed to migrate database:", err)
57+
}
58+
}
59+
5160
// Initialize database.
5261
db, err := model.OpenDB(context.Background(), cfg.Database.DSN, 0, 0)
5362
if err != nil {

0 commit comments

Comments
 (0)