Repository navigation
db_bench-run #72
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Triggered after db_bench-build. Runs ToplingDB + RocksDB v8.10 + RocksDB master, | |
| # emits comparison tables, publishes Pages. | |
| name: "db_bench-run" | |
| on: | |
| workflow_run: | |
| workflows: ["db_bench-build"] | |
| types: [completed] | |
| workflow_dispatch: | |
| inputs: | |
| build_run_id: | |
| description: Build workflow run id whose artifact to download | |
| required: true | |
| type: string | |
| value_size: | |
| description: db_bench -value_size bytes (default 50) | |
| required: false | |
| default: "50" | |
| type: string | |
| num: | |
| description: db_bench -num keys per pass (default 50000000) | |
| required: false | |
| default: "50000000" | |
| type: string | |
| max_background_compactions: | |
| description: >- | |
| max_background_compactions (auto = max(1,min(13,ceil(nproc/2)))). | |
| required: false | |
| default: auto | |
| type: string | |
| cached_pages_use_sys: | |
| description: >- | |
| If true, drop page cache before each sampled bench and set | |
| SYS_CACHED_OF_EMPTY from /proc/meminfo Cached so pagecache is | |
| system-wide Cached growth (pages). Default: per-fd cachestat. | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: pages-deploy | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| pages: write | |
| id-token: write | |
| actions: read | |
| jobs: | |
| bench-and-publish: | |
| if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| VARIANT: plain | |
| ARTIFACT_NAME: db_bench-bin-plain | |
| VALUE_SIZE: ${{ inputs.value_size || '50' }} | |
| NUM: ${{ inputs.num || '50000000' }} | |
| MAX_BACKGROUND_COMPACTIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.max_background_compactions || 'auto' }} | |
| CACHED_PAGES_USE_SYS: ${{ github.event_name == 'workflow_dispatch' && inputs.cached_pages_use_sys || false }} | |
| steps: | |
| - name: Checkout build commit | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} | |
| - name: Install runtime dependencies | |
| run: | | |
| set -xe | |
| sudo apt-get update -y && sudo apt-get install -y \ | |
| gcc \ | |
| libjemalloc-dev libaio-dev libgflags-dev zlib1g-dev \ | |
| libbz2-dev libcurl4-openssl-dev liburing-dev \ | |
| libsnappy-dev liblz4-dev libzstd-dev | |
| chmod +x .github/scripts/ensure_sample_statm_fdcache.sh \ | |
| .github/scripts/run_sample_statm_fdcache.sh | |
| .github/scripts/ensure_sample_statm_fdcache.sh | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACT_NAME }} | |
| run-id: ${{ github.event_name == 'workflow_dispatch' && inputs.build_run_id || github.event.workflow_run.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| path: _artifact | |
| - name: Unpack bench bundle | |
| run: | | |
| set -xe | |
| RUN_PREFIX="$PWD/_run_prefix" | |
| rm -rf "$RUN_PREFIX" logs | |
| mkdir -p "$RUN_PREFIX" logs | |
| tar -C "$RUN_PREFIX" -xzf _artifact/bench-prefix.tgz | |
| test -d "$RUN_PREFIX/topling" | |
| test -d "$RUN_PREFIX/rocksdb-v8.10" | |
| test -d "$RUN_PREFIX/rocksdb-master" | |
| test -x "$RUN_PREFIX/topling/bin/db_bench" | |
| test -f "$RUN_PREFIX/topling/toplingdb-conf/db_bench_enterprise_zipkeyonly.yaml" | |
| test -f "$RUN_PREFIX/topling/toplingdb-conf/db_bench_enterprise_zipkeyvalue.yaml" | |
| test -x "$RUN_PREFIX/rocksdb-v8.10/bin/db_bench" | |
| test -x "$RUN_PREFIX/rocksdb-master/bin/db_bench" | |
| echo "RUN_PREFIX=$RUN_PREFIX" >> "$GITHUB_ENV" | |
| - name: Probe /dev/shm | |
| run: | | |
| set -xe | |
| # Engines rotate exclusive use of /dev/shm (one DB at a time). | |
| # Do not pre-create all three DB dirs here. | |
| df -h /dev/shm | |
| - name: Run ToplingDB benches | |
| run: | | |
| set -xeo pipefail | |
| mkdir -p logs | |
| { | |
| echo "num=${NUM}" | |
| echo "key_size=8" | |
| echo "value_size=${VALUE_SIZE}" | |
| } > logs/bench_settings.txt | |
| TOPLING="$RUN_PREFIX/topling" | |
| export LD_LIBRARY_PATH="$TOPLING/lib:${LD_LIBRARY_PATH:-}" | |
| ldd "$TOPLING/bin/db_bench" | |
| export TOPLINGDB_GetContext_sampling=kNone | |
| export ROCKSDB_KICK_OUT_OPTIONS_FILE=1 | |
| DB_PATH=/dev/shm/db_bench_enterprise | |
| YAML_ZKO="$TOPLING/toplingdb-conf/db_bench_enterprise_zipkeyonly.yaml" | |
| YAML_ZKV="$TOPLING/toplingdb-conf/db_bench_enterprise_zipkeyvalue.yaml" | |
| test -f "$YAML_ZKO" | |
| test -f "$YAML_ZKV" | |
| NPROC="$(nproc)" | |
| if [ -z "${MAX_BACKGROUND_COMPACTIONS:-}" ] || [ "$MAX_BACKGROUND_COMPACTIONS" = auto ]; then | |
| MBC="$(awk -v n="$NPROC" 'BEGIN { | |
| w = n - n/2; m = int(w); if (m < w) m++; | |
| if (m < 1) m = 1; if (m > 13) m = 13; print m | |
| }')" | |
| else | |
| MBC="$MAX_BACKGROUND_COMPACTIONS" | |
| case "$MBC" in | |
| *[!0-9]*|'') echo "FAIL: max_background_compactions must be a positive integer or auto, got $MBC" >&2; exit 1 ;; | |
| esac | |
| REQ_MBC="$MBC" | |
| if [ "$MBC" -lt 1 ]; then MBC=1; fi | |
| if [ "$MBC" -gt 13 ]; then MBC=13; fi | |
| if [ "$MBC" -gt "$NPROC" ]; then MBC="$NPROC"; fi | |
| if [ "$REQ_MBC" != "$MBC" ]; then | |
| echo "max_background_compactions clamped ${REQ_MBC} -> ${MBC} (nproc=${NPROC}, max=13)" | |
| fi | |
| fi | |
| echo "graft mbc: nproc=${NPROC} max_background_compactions=${MBC}" | |
| WORK_ZKO="$(mktemp)" | |
| WORK_ZKV="$(mktemp)" | |
| python3 .github/scripts/graft_bench_yaml.py \ | |
| --set-max-background-compactions "$MBC" \ | |
| --out "$WORK_ZKO" \ | |
| "$YAML_ZKO" | |
| python3 .github/scripts/graft_bench_yaml.py \ | |
| --set-max-background-compactions "$MBC" \ | |
| --out "$WORK_ZKV" \ | |
| "$YAML_ZKV" | |
| prepare_db() { | |
| rm -rf "$DB_PATH" | |
| mkdir -p "$DB_PATH" | |
| if [ -d "$TOPLING/site" ]; then | |
| cp -a "$TOPLING/site/." "$DB_PATH/" | |
| fi | |
| } | |
| # eng_id log dir name; yaml path; run_memtable 0|1 | |
| run_topling_suite() { | |
| local eng_id="$1" | |
| local yaml="$2" | |
| local run_memtable="$3" | |
| local logdir="logs/${eng_id}" | |
| mkdir -p "$logdir" | |
| test -f "$yaml" | |
| record_shm() { | |
| local label="$1" | |
| { | |
| echo "apparent_bytes=$(du -sb "$DB_PATH" | awk '{print $1}')" | |
| echo "allocated_bytes=$(du -s -B1 "$DB_PATH" | awk '{print $1}')" | |
| } | tee "${logdir}/shm_usage-${label}.txt" | |
| du -sh --apparent-size "$DB_PATH" || true | |
| du -sh "$DB_PATH" || true | |
| df -h /dev/shm | |
| } | |
| record_rss() { | |
| local label="$1" | |
| local time_file="${logdir}/time-${label}.txt" | |
| if [ -f "$time_file" ]; then | |
| local kb | |
| kb=$(grep -oP 'max_rss_kb=\K\d+' "$time_file") | |
| echo "max_rss_bytes=$((kb * 1024))" > "${logdir}/rss_usage-${label}.txt" | |
| fi | |
| } | |
| save_db_log() { | |
| local label="$1" | |
| test -f "$DB_PATH/LOG" || { echo "FAIL: missing $DB_PATH/LOG after ${label}" >&2; return 1; } | |
| cp -f "$DB_PATH/LOG" "${logdir}/LOG-${label}" | |
| } | |
| # Pass 1: fillrandom — prefix 3 simple (keep L3..L6 from prebake). | |
| prepare_db | |
| yaml_fr="${logdir}/db_bench-fillrandom.yaml" | |
| python3 .github/scripts/graft_bench_yaml.py \ | |
| --prefix-level-writers 3 simple \ | |
| --out "$yaml_fr" \ | |
| "$yaml" | |
| args_fr=( | |
| -json "$yaml_fr" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom | |
| -enable_zero_copy | |
| -progress_reports=false | |
| -report_bench_start_time | |
| -compact_target_level=6 | |
| ) | |
| .github/scripts/run_sample_statm_fdcache.sh "${logdir}/statm_series-fillrandom.txt" "${logdir}/time-fillrandom.txt" \ | |
| "$TOPLING/bin/db_bench" "${args_fr[@]}" \ | |
| >"${logdir}/db_bench-fillrandom.log" 2>&1 | |
| cat "${logdir}/db_bench-fillrandom.log" | |
| save_db_log fillrandom | |
| args_omit=( | |
| -json "$yaml_fr" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq | |
| -scan_omit_key -scan_omit_value | |
| -use_existing_db=1 | |
| -enable_zero_copy | |
| -progress_reports=false | |
| ) | |
| /usr/bin/time -f 'max_rss_kb=%M' -o "${logdir}/time-fillrandom-omit.txt" -- \ | |
| "$TOPLING/bin/db_bench" "${args_omit[@]}" >"${logdir}/db_bench-fillrandom-omit.log" 2>&1 | |
| cat "${logdir}/db_bench-fillrandom-omit.log" | |
| save_db_log fillrandom-omit | |
| record_rss fillrandom | |
| record_rss fillrandom-omit | |
| record_shm fillrandom | |
| rm -rf "$DB_PATH" | |
| # Pass 2: fillseq — prefix 6 zipkeyonly (keep L6 from prebake). | |
| prepare_db | |
| yaml_fs="${logdir}/db_bench-fillseq.yaml" | |
| python3 .github/scripts/graft_bench_yaml.py \ | |
| --prefix-level-writers 6 zipkeyonly \ | |
| --target-file-size-base 128M \ | |
| --target-file-size-multiplier 1 \ | |
| --out "$yaml_fs" \ | |
| "$yaml" | |
| args=( | |
| -json "$yaml_fs" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom | |
| -enable_zero_copy | |
| -progress_reports=false | |
| -report_bench_start_time | |
| -compact_target_level=6 | |
| ) | |
| .github/scripts/run_sample_statm_fdcache.sh "${logdir}/statm_series-fillseq.txt" "${logdir}/time-fillseq.txt" \ | |
| "$TOPLING/bin/db_bench" "${args[@]}" \ | |
| >"${logdir}/db_bench.log" 2>&1 | |
| cat "${logdir}/db_bench.log" | |
| save_db_log fillseq | |
| args_omit_fs=( | |
| -json "$yaml_fs" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq | |
| -scan_omit_key -scan_omit_value | |
| -use_existing_db=1 | |
| -enable_zero_copy | |
| -progress_reports=false | |
| ) | |
| /usr/bin/time -f 'max_rss_kb=%M' -o "${logdir}/time-fillseq-omit.txt" -- \ | |
| "$TOPLING/bin/db_bench" "${args_omit_fs[@]}" >"${logdir}/db_bench-fillseq-omit.log" 2>&1 | |
| cat "${logdir}/db_bench-fillseq-omit.log" | |
| save_db_log fillseq-omit | |
| record_rss fillseq | |
| record_rss fillseq-omit | |
| record_shm fillseq | |
| if [ "$run_memtable" = "1" ]; then | |
| mt=( | |
| -benchmarks=fillrandom,readrandom | |
| -item_size=0 | |
| -num_operations=100000000 | |
| -write_buffer_size=536870912 | |
| ) | |
| "$TOPLING/bin/memtablerep_bench" "${mt[@]}" -memtablerep=skiplist \ | |
| 2>&1 | tee "${logdir}/memtablerep_bench-skiplist.log" | |
| "$TOPLING/bin/memtablerep_bench" "${mt[@]}" \ | |
| -memtablerep='cspp:{"mem_cap":"16G","use_hugepage":false}' \ | |
| 2>&1 | tee "${logdir}/memtablerep_bench-cspp.log" | |
| fi | |
| rm -rf "$DB_PATH" | |
| } | |
| run_topling_suite zipkeyonly "$WORK_ZKO" 1 | |
| run_topling_suite zipkeyvalue "$WORK_ZKV" 0 | |
| rm -f "$WORK_ZKO" "$WORK_ZKV" | |
| for f in logs/zipkeyonly/db_bench*.log logs/zipkeyonly/memtablerep_bench*.log \ | |
| logs/zipkeyvalue/db_bench*.log; do | |
| test -s "$f" || { echo "empty bench log: $f" >&2; exit 1; } | |
| grep -Eiq 'ops/sec|us/op|micros/op|Elapsed time' "$f" \ | |
| || { echo "bench log lacks metrics: $f" >&2; exit 1; } | |
| done | |
| - name: Compute RocksDB cache_size | |
| run: | | |
| set -xe | |
| mem_total_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}') | |
| cache_size_bytes=$(( mem_total_kb * 1024 / 2 )) | |
| echo "ROCKSDB_CACHE_SIZE=$cache_size_bytes" >> "$GITHUB_ENV" | |
| echo "MEM_TOTAL_BYTES=$(( mem_total_kb * 1024 ))" >> "$GITHUB_ENV" | |
| echo "cache_size_bytes=$cache_size_bytes ($(numfmt --to=iec-i --suffix=B $cache_size_bytes))" | |
| - name: Collect runner environment | |
| run: | | |
| set -xe | |
| { | |
| echo "os_pretty_name=$(grep -oP 'PRETTY_NAME="\K[^"]+' /etc/os-release || echo unknown)" | |
| echo "kernel=$(uname -r)" | |
| echo "cpu_model=$(lscpu | grep -oP 'Model name:\s+\K.*')" | |
| echo "cpu_count=$(nproc)" | |
| echo "mem_total_bytes=$MEM_TOTAL_BYTES" | |
| echo "shm_size_bytes=$(df -B1 /dev/shm | awk 'NR==2{print $2}')" | |
| } | tee logs/runner_env.txt | |
| - name: Run RocksDB v8.10 benches | |
| run: | | |
| set -xeo pipefail | |
| ENG="$RUN_PREFIX/rocksdb-v8.10" | |
| export LD_LIBRARY_PATH="$ENG/lib:${LD_LIBRARY_PATH:-}" | |
| ldd "$ENG/bin/db_bench" || true | |
| mkdir -p logs/rocksdb-v8.10 | |
| DB_PATH=/dev/shm/db_bench_rocksdb_v810 | |
| LOGDIR=logs/rocksdb-v8.10 | |
| record_shm() { | |
| local label="$1" | |
| { | |
| echo "apparent_bytes=$(du -sb "$DB_PATH" | awk '{print $1}')" | |
| echo "allocated_bytes=$(du -s -B1 "$DB_PATH" | awk '{print $1}')" | |
| } | tee "${LOGDIR}/shm_usage-${label}.txt" | |
| du -sh --apparent-size "$DB_PATH" || true | |
| du -sh "$DB_PATH" || true | |
| df -h /dev/shm | |
| } | |
| record_rss() { | |
| local label="$1" | |
| local time_file="${LOGDIR}/time-${label}.txt" | |
| if [ -f "$time_file" ]; then | |
| local kb | |
| kb=$(grep -oP 'max_rss_kb=\K\d+' "$time_file") | |
| echo "max_rss_bytes=$((kb * 1024))" > "${LOGDIR}/rss_usage-${label}.txt" | |
| fi | |
| } | |
| save_db_log() { | |
| local label="$1" | |
| test -f "$DB_PATH/LOG" || { echo "FAIL: missing $DB_PATH/LOG after ${label}" >&2; return 1; } | |
| cp -f "$DB_PATH/LOG" "${LOGDIR}/LOG-${label}" | |
| } | |
| echo "cache_size_bytes=$ROCKSDB_CACHE_SIZE" > "${LOGDIR}/bench_settings.txt" | |
| # Pass 1: fillrandom + compact. | |
| rm -rf "$DB_PATH" | |
| mkdir -p "$DB_PATH" | |
| args_fr=( | |
| -db="$DB_PATH" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -cache_size="$ROCKSDB_CACHE_SIZE" | |
| -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom | |
| -progress_reports=false | |
| -report_bench_start_time | |
| -compact_target_level=6 | |
| ) | |
| .github/scripts/run_sample_statm_fdcache.sh "${LOGDIR}/statm_series-fillrandom.txt" "${LOGDIR}/time-fillrandom.txt" \ | |
| "$ENG/bin/db_bench" "${args_fr[@]}" \ | |
| >"${LOGDIR}/db_bench-fillrandom.log" 2>&1 | |
| cat "${LOGDIR}/db_bench-fillrandom.log" | |
| record_rss fillrandom | |
| # RocksDB has no omit API; derive omit RSS from main pass | |
| cp "${LOGDIR}/rss_usage-fillrandom.txt" "${LOGDIR}/rss_usage-fillrandom-omit.txt" 2>/dev/null || true | |
| record_shm fillrandom | |
| save_db_log fillrandom | |
| rm -rf "$DB_PATH" | |
| # Pass 2: existing fillseq suite. | |
| mkdir -p "$DB_PATH" | |
| args=( | |
| -db="$DB_PATH" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -cache_size="$ROCKSDB_CACHE_SIZE" | |
| -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom | |
| -progress_reports=false | |
| -report_bench_start_time | |
| -compact_target_level=6 | |
| ) | |
| .github/scripts/run_sample_statm_fdcache.sh "${LOGDIR}/statm_series-fillseq.txt" "${LOGDIR}/time-fillseq.txt" \ | |
| "$ENG/bin/db_bench" "${args[@]}" \ | |
| >"${LOGDIR}/db_bench.log" 2>&1 | |
| cat "${LOGDIR}/db_bench.log" | |
| record_rss fillseq | |
| # RocksDB has no omit API; derive omit RSS from main pass | |
| cp "${LOGDIR}/rss_usage-fillseq.txt" "${LOGDIR}/rss_usage-fillseq-omit.txt" 2>/dev/null || true | |
| record_shm fillseq | |
| save_db_log fillseq | |
| mt=( | |
| -benchmarks=fillrandom,readrandom | |
| -item_size=0 | |
| -num_operations=100000000 | |
| -write_buffer_size=536870912 | |
| ) | |
| "$ENG/bin/memtablerep_bench" "${mt[@]}" -memtablerep=skiplist \ | |
| 2>&1 | tee "${LOGDIR}/memtablerep_bench-skiplist.log" | |
| for f in "${LOGDIR}"/db_bench*.log \ | |
| "${LOGDIR}"/memtablerep_bench*.log; do | |
| test -s "$f" || { echo "empty bench log: $f" >&2; exit 1; } | |
| grep -Eiq 'ops/sec|us/op|micros/op|Elapsed time' "$f" \ | |
| || { echo "bench log lacks metrics: $f" >&2; exit 1; } | |
| done | |
| rm -rf "$DB_PATH" | |
| - name: Run RocksDB master benches | |
| run: | | |
| set -xeo pipefail | |
| ENG="$RUN_PREFIX/rocksdb-master" | |
| export LD_LIBRARY_PATH="$ENG/lib:${LD_LIBRARY_PATH:-}" | |
| ldd "$ENG/bin/db_bench" || true | |
| mkdir -p logs/rocksdb-master | |
| DB_PATH=/dev/shm/db_bench_rocksdb_master | |
| LOGDIR=logs/rocksdb-master | |
| record_shm() { | |
| local label="$1" | |
| { | |
| echo "apparent_bytes=$(du -sb "$DB_PATH" | awk '{print $1}')" | |
| echo "allocated_bytes=$(du -s -B1 "$DB_PATH" | awk '{print $1}')" | |
| } | tee "${LOGDIR}/shm_usage-${label}.txt" | |
| du -sh --apparent-size "$DB_PATH" || true | |
| du -sh "$DB_PATH" || true | |
| df -h /dev/shm | |
| } | |
| record_rss() { | |
| local label="$1" | |
| local time_file="${LOGDIR}/time-${label}.txt" | |
| if [ -f "$time_file" ]; then | |
| local kb | |
| kb=$(grep -oP 'max_rss_kb=\K\d+' "$time_file") | |
| echo "max_rss_bytes=$((kb * 1024))" > "${LOGDIR}/rss_usage-${label}.txt" | |
| fi | |
| } | |
| save_db_log() { | |
| local label="$1" | |
| test -f "$DB_PATH/LOG" || { echo "FAIL: missing $DB_PATH/LOG after ${label}" >&2; return 1; } | |
| cp -f "$DB_PATH/LOG" "${LOGDIR}/LOG-${label}" | |
| } | |
| echo "cache_size_bytes=$ROCKSDB_CACHE_SIZE" > "${LOGDIR}/bench_settings.txt" | |
| # Pass 1: fillrandom + compact. | |
| rm -rf "$DB_PATH" | |
| mkdir -p "$DB_PATH" | |
| args_fr=( | |
| -db="$DB_PATH" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -cache_size="$ROCKSDB_CACHE_SIZE" | |
| -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom | |
| -progress_reports=false | |
| -report_bench_start_time | |
| -compact_target_level=6 | |
| ) | |
| .github/scripts/run_sample_statm_fdcache.sh "${LOGDIR}/statm_series-fillrandom.txt" "${LOGDIR}/time-fillrandom.txt" \ | |
| "$ENG/bin/db_bench" "${args_fr[@]}" \ | |
| >"${LOGDIR}/db_bench-fillrandom.log" 2>&1 | |
| cat "${LOGDIR}/db_bench-fillrandom.log" | |
| record_rss fillrandom | |
| # RocksDB has no omit API; omit RSS is the same scan as readseq. | |
| cp "${LOGDIR}/rss_usage-fillrandom.txt" "${LOGDIR}/rss_usage-fillrandom-omit.txt" 2>/dev/null || true | |
| record_shm fillrandom | |
| save_db_log fillrandom | |
| rm -rf "$DB_PATH" | |
| # Pass 2: existing fillseq suite. | |
| mkdir -p "$DB_PATH" | |
| args=( | |
| -db="$DB_PATH" | |
| -num="${NUM}" | |
| -key_size=8 | |
| -value_size="${VALUE_SIZE}" | |
| -batch_size=1000 | |
| -cache_size="$ROCKSDB_CACHE_SIZE" | |
| -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom | |
| -progress_reports=false | |
| -report_bench_start_time | |
| -compact_target_level=6 | |
| ) | |
| .github/scripts/run_sample_statm_fdcache.sh "${LOGDIR}/statm_series-fillseq.txt" "${LOGDIR}/time-fillseq.txt" \ | |
| "$ENG/bin/db_bench" "${args[@]}" \ | |
| >"${LOGDIR}/db_bench.log" 2>&1 | |
| cat "${LOGDIR}/db_bench.log" | |
| record_rss fillseq | |
| # RocksDB has no omit API; omit RSS is the same scan as readseq. | |
| cp "${LOGDIR}/rss_usage-fillseq.txt" "${LOGDIR}/rss_usage-fillseq-omit.txt" 2>/dev/null || true | |
| record_shm fillseq | |
| save_db_log fillseq | |
| mt=( | |
| -benchmarks=fillrandom,readrandom | |
| -item_size=0 | |
| -num_operations=100000000 | |
| -write_buffer_size=536870912 | |
| ) | |
| "$ENG/bin/memtablerep_bench" "${mt[@]}" -memtablerep=skiplist \ | |
| 2>&1 | tee "${LOGDIR}/memtablerep_bench-skiplist.log" | |
| for f in "${LOGDIR}"/db_bench*.log \ | |
| "${LOGDIR}"/memtablerep_bench*.log; do | |
| test -s "$f" || { echo "empty bench log: $f" >&2; exit 1; } | |
| grep -Eiq 'ops/sec|us/op|micros/op|Elapsed time' "$f" \ | |
| || { echo "bench log lacks metrics: $f" >&2; exit 1; } | |
| done | |
| rm -rf "$DB_PATH" | |
| - name: Upload DB INFO LOGs and bench yamls (Actions artifact) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: db-info-logs-${{ env.VARIANT }} | |
| path: | | |
| logs/*/LOG-* | |
| logs/*/db_bench*.yaml | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Emit pages fragment | |
| run: | | |
| set -xe | |
| python3 .github/scripts/bench_logs_to_pages.py emit \ | |
| --variant "$VARIANT" \ | |
| --run-id "$GITHUB_RUN_ID" \ | |
| --log-root logs \ | |
| --engine-meta-root "$RUN_PREFIX" \ | |
| --actions-run-url "https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ | |
| --out _pages | |
| - name: Checkout gh-pages | |
| id: checkout_pages | |
| continue-on-error: true | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: _gh_pages | |
| - name: Init gh-pages if missing | |
| if: steps.checkout_pages.outcome == 'failure' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -xe | |
| rm -rf _gh_pages | |
| mkdir -p _gh_pages | |
| cd _gh_pages | |
| git init | |
| git checkout -b gh-pages | |
| echo '[]' > history.json | |
| touch .nojekyll | |
| printf '%s\n' '<!DOCTYPE html><html><body><h1>ToplingDB vs RocksDB</h1><p>empty</p></body></html>' > index.html | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add . | |
| git commit -m "Initialize gh-pages for db_bench results" | |
| git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git push -u origin gh-pages | |
| - name: Merge into gh-pages tree | |
| run: | | |
| set -xe | |
| python3 .github/scripts/bench_logs_to_pages.py merge \ | |
| --merge-into _gh_pages \ | |
| --from _pages \ | |
| --variant "$VARIANT" | |
| - name: Push gh-pages | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -xe | |
| cd _gh_pages | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git add -A | |
| if git diff --cached --quiet; then | |
| echo "No site changes to commit" | |
| else | |
| git commit -m "db_bench ${VARIANT} comparison from run ${GITHUB_RUN_ID}" | |
| fi | |
| git push origin HEAD:gh-pages | |
| - name: Prepare Pages payload without .git | |
| run: | | |
| set -xe | |
| rm -rf _pages_payload | |
| mkdir -p _pages_payload | |
| tar -C _gh_pages --exclude .git -cf - . | tar -C _pages_payload -xf - | |
| - name: Upload Pages artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: _pages_payload | |
| deploy: | |
| needs: bench-and-publish | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v4 |