Skip to content

Add script to take backup and support merge in restore #312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ArtyomSavchenko merged 1 commit into from
Jul 13, 2026
Merged

Add script to take backup and support merge in restore #312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions backup-create.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
#!/usr/bin/env bash
#
# Take a backup of a workspace in this local Huly deployment.
# Connection parameters are read from huly_v7.conf (same file docker compose uses).
#
# Backups are incremental: each run only downloads and stores documents that
# are new or changed since the last backup snapshot in <backup-dir> (based on
# per-domain hashes). Re-running this against the same directory is cheap and
# safe - it just adds another snapshot on top of what's already there.
#
# Usage:
# ./backup-create.sh <backup-dir> <workspace> [options] [-- <extra tool args>]
#
# Options:
# --full Force a full recheck: compare every document instead of relying on
# stored hashes (slower, use if you suspect the backup is out of sync)
# --full-verify Full verification: download and diff every document against the
# server, not just changed ones (slowest, most thorough)
# --force Force backup even if no transactions changed since last run
# --include <domains> ; separated list of domains to include (default: all)
# --skip <domains> ; separated list of domains to skip
# --blob-limit <mb> Skip blobs larger than this size in MB (default: 5)
# --content-types <ct> ; separated content type prefixes to skip downloading (e.g. video/;audio/)
# --timeout <sec> Connect timeout in seconds (default: 30)
# --keep-snapshots <n> Compact backup once it has more than N snapshots (default: 14)
#
# Examples:
# ./backup-create.sh ./backups/myws myws
# ./backup-create.sh ./backups/myws myws --full-verify
# ./backup-create.sh ./backups/myws myws --skip fulltext-blob
set -euo pipefail

CONFIG_FILE="huly_v7.conf"
if [ ! -f "$CONFIG_FILE" ]; then
echo -e "\033[1;31mConfig not found: $CONFIG_FILE. Run ./setup.sh first.\033[0m"
exit 1
fi
# shellcheck disable=SC1090
source "$CONFIG_FILE"

# Args
BACKUP_DIR="${1:-}"
WORKSPACE="${2:-}"
shift $(( $# >= 2 ? 2 : $# )) || true

FULL=false
FULL_VERIFY=false
FORCE=false
INCLUDE=""
SKIP=""
BLOB_LIMIT="5"
CONTENT_TYPES=""
TIMEOUT="30"
KEEP_SNAPSHOTS="14"
EXTRA_ARGS=()

while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
case "$1" in
--full) FULL=true; shift ;;
--full-verify) FULL_VERIFY=true; shift ;;
--force) FORCE=true; shift ;;
--include) INCLUDE="$2"; shift 2 ;;
--skip) SKIP="$2"; shift 2 ;;
--blob-limit) BLOB_LIMIT="$2"; shift 2 ;;
--content-types) CONTENT_TYPES="$2"; shift 2 ;;
--timeout) TIMEOUT="$2"; shift 2 ;;
--keep-snapshots) KEEP_SNAPSHOTS="$2"; shift 2 ;;
*)
echo -e "\033[1;31mUnknown option: $1\033[0m"
exit 1
;;
esac
done
if [ "${1:-}" == "--" ]; then
shift
EXTRA_ARGS=("$@")
fi

if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then
echo "Usage: $0 <backup-dir> <workspace> [options] [-- <extra tool args>]"
echo ""
echo " <backup-dir> Local directory to store/append the backup (created if missing)"
echo " <workspace> Workspace id/url to back up"
echo " --full Force full recheck of all documents (slower)"
echo " --full-verify Download and diff every document against the server (slowest)"
echo " --force Force backup even if no transactions changed"
echo " --include <d> ; separated list of domains to include (default: all)"
echo " --skip <d> ; separated list of domains to skip"
echo " --blob-limit <mb> Skip blobs larger than this size in MB (default: 5)"
echo " --content-types <t> ; separated content type prefixes to skip (e.g. video/;audio/)"
echo " --timeout <sec> Connect timeout in seconds (default: 30)"
echo " --keep-snapshots <n> Compact once more than N snapshots exist (default: 14)"
echo " -- <args> Extra args passed to 'tool backup'"
exit 1
fi

# Create the backup directory if it doesn't exist yet (first run)
mkdir -p "$BACKUP_DIR"
BACKUP_ABS="$(cd "$BACKUP_DIR" && pwd)"

# Required config values
: "${SECRET:?SECRET missing in $CONFIG_FILE}"
: "${CR_DB_URL:?CR_DB_URL missing in $CONFIG_FILE}"
: "${HULY_VERSION:?HULY_VERSION missing in $CONFIG_FILE}"
: "${DOCKER_NAME:?DOCKER_NAME missing in $CONFIG_FILE}"

NETWORK="${DOCKER_NAME}_huly_net"

echo -e "\033[1;34mCreating backup:\033[0m"
echo " Destination: $BACKUP_ABS"
echo " Workspace: $WORKSPACE"
echo " Full recheck: $FULL"
echo " Full verify: $FULL_VERIFY"
echo " Force: $FORCE"
echo " Include: ${INCLUDE:-*}"
echo " Skip: ${SKIP:-none}"
echo " Blob limit: ${BLOB_LIMIT}mb"
echo " Keep snapshots: $KEEP_SNAPSHOTS"
echo " Network: $NETWORK"
echo " Version: $HULY_VERSION"
echo ""

# Verify the stack network exists (stack must be up)
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
echo -e "\033[1;31mNetwork $NETWORK not found. Start the stack first: docker compose up -d\033[0m"
exit 1
fi

CMD=(backup /backup "$WORKSPACE")
[ "$FULL" == true ] && CMD+=(--full)
[ "$FULL_VERIFY" == true ] && CMD+=(--fullVerify)
[ "$FORCE" == true ] && CMD+=(--force)
[ -n "$INCLUDE" ] && CMD+=(--include "$INCLUDE")
[ -n "$SKIP" ] && CMD+=(--skip "$SKIP")
[ -n "$BLOB_LIMIT" ] && CMD+=(--blobLimit "$BLOB_LIMIT")
[ -n "$CONTENT_TYPES" ] && CMD+=(--contentTypes "$CONTENT_TYPES")
[ -n "$TIMEOUT" ] && CMD+=(--timeout "$TIMEOUT")
[ -n "$KEEP_SNAPSHOTS" ] && CMD+=(--keepSnapshots "$KEEP_SNAPSHOTS")
[ ${#EXTRA_ARGS[@]} -gt 0 ] && CMD+=("${EXTRA_ARGS[@]}")

RUN_TOOL_DOCKER_ARGS="-v ${BACKUP_ABS}:/backup" ./run-tool.sh "${CMD[@]}"

echo -e "\n\033[1;32mBackup finished.\033[0m"
echo " Stored in: $BACKUP_ABS"
42 changes: 39 additions & 3 deletions backup-restore.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,14 @@
# Restore a downloaded backup into this local Huly deployment.
# Connection parameters are read from huly_v7.conf (same file docker compose uses).
#
# By default this restores in MERGE mode: only new/changed documents from the
# backup are uploaded, and any data already present in the workspace but not
# in the backup is left untouched (nothing is deleted). Use --no-merge to
# restore the destructive way (server is made to match the backup exactly,
# i.e. anything not in the backup gets removed).
#
# Usage:
# ./backup-restore.sh <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [-- <extra tool args>]
# ./backup-restore.sh <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [--no-merge] [-y] [-- <extra tool args>]
#
# Example:
# ./backup-restore.sh ./backups/myws myws
Expand All @@ -29,6 +35,14 @@ RESTORE_ACCOUNTS=true
# Upgrade the workspace to the current model version after restore. On by default -
# the backup is usually from an older version. Disable with --no-upgrade.
UPGRADE=true
# Merge mode: only add/update documents from the backup, never delete anything
# already present in the workspace. On by default so restores are safe to
# re-run and only bring in what's missing. Disable with --no-merge to make
# the workspace match the backup exactly (deletes data not in the backup).
MERGE=true
# Skip the interactive confirmation prompt that --no-merge triggers (useful
# for CI/automation). Has no effect when merge is on.
ASSUME_YES=false

shift $(( $# >= 2 ? 2 : $# )) || true

Expand All @@ -39,6 +53,9 @@ while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
--no-accounts) RESTORE_ACCOUNTS=false; shift ;;
--upgrade) UPGRADE=true; shift ;;
--no-upgrade) UPGRADE=false; shift ;;
--merge) MERGE=true; shift ;;
--no-merge) MERGE=false; shift ;;
-y|--yes) ASSUME_YES=true; shift ;;
*) DATE="$1"; shift ;;
esac
done
Expand All @@ -48,14 +65,16 @@ if [ "${1:-}" == "--" ]; then
fi

if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then
echo "Usage: $0 <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [-- <extra tool args>]"
echo "Usage: $0 <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [--no-merge] [-y] [-- <extra tool args>]"
echo ""
echo " <backup-dir> Local directory with downloaded backup files"
echo " <workspace> Target workspace id/url to restore into"
echo " [date] Optional snapshot timestamp (ms). Default: latest"
echo " --no-accounts Do not restore person/socialId accounts (default: restore them)"
echo " --no-upgrade Do not upgrade the workspace after restore (default: upgrade)"
echo " -- <args> Extra args passed to 'tool backup-restore' (e.g. --merge)"
echo " --no-merge Delete data not present in the backup (default: merge, nothing is deleted)"
echo " -y, --yes Skip the confirmation prompt triggered by --no-merge"
echo " -- <args> Extra args passed to 'tool backup-restore' (e.g. --recheck)"
exit 1
fi

Expand All @@ -81,6 +100,7 @@ echo " Workspace: $WORKSPACE"
echo " Date: ${DATE:-latest}"
echo " Accounts: ${RESTORE_ACCOUNTS}"
echo " Upgrade: ${UPGRADE}"
echo " Merge: ${MERGE} $([ "$MERGE" == true ] && echo '(existing data preserved, only new/changed restored)' || echo '(destructive, matches backup exactly)')"
echo " Network: $NETWORK"
echo " Version: $HULY_VERSION"
echo ""
Expand All @@ -91,9 +111,25 @@ if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
exit 1
fi

# --no-merge deletes any data in the workspace that isn't present in the
# backup and cannot be undone - require explicit confirmation unless -y/--yes
# was passed (e.g. for scripted/CI use).
if [ "$MERGE" != true ] && [ "$ASSUME_YES" != true ]; then
echo -e "\033[1;31mWARNING: --no-merge will DELETE any data in workspace '$WORKSPACE'\033[0m"
echo -e "\033[1;31mthat is not present in the backup. This cannot be undone.\033[0m"
echo ""
read -r -p "Type the workspace name (${WORKSPACE}) to confirm, anything else to abort: " CONFIRM
if [ "$CONFIRM" != "$WORKSPACE" ]; then
echo "Aborted."
exit 1
fi
echo ""
fi

CMD=(backup-restore /backup "$WORKSPACE")
[ -n "$DATE" ] && CMD+=("$DATE")
[ "$RESTORE_ACCOUNTS" == true ] && CMD+=(--accounts)
[ "$MERGE" == true ] && CMD+=(--merge)
[ ${#EXTRA_ARGS[@]} -gt 0 ] && CMD+=("${EXTRA_ARGS[@]}")

RUN_TOOL_DOCKER_ARGS="-v ${BACKUP_ABS}:/backup" ./run-tool.sh "${CMD[@]}"
Expand Down
24 changes: 22 additions & 2 deletions restore-workspace.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,15 @@
# --skip-account Don't create the account (it already exists)
# --skip-workspace Don't create/assign the workspace (it already exists)
# --date <ms> Backup snapshot timestamp, passed to backup-restore.sh
# --merge Merge mode: keep existing data, only add/update from backup (default)
# --no-merge Destructive mode: make the workspace match the backup exactly
# (prompts for confirmation unless -y/--yes is also given)
# -y, --yes Skip the confirmation prompt triggered by --no-merge
#
# Examples:
# ./restore-workspace.sh ./backups/myws myws
# ./restore-workspace.sh ./backups/myws myws -e me@example.com -p secret
# ./restore-workspace.sh ./backups/myws myws --skip-account -- --merge
# ./restore-workspace.sh ./backups/myws myws --skip-account --no-merge
set -euo pipefail

CONFIG_FILE="huly_v7.conf"
Expand All @@ -41,6 +45,13 @@ SKIP_ACCOUNT=false
SKIP_WORKSPACE=false
DATE=""
EXTRA_ARGS=()
# Merge mode: only add/update documents from the backup, never delete anything
# already present in the workspace. On by default. Disable with --no-merge to
# make the workspace match the backup exactly (deletes data not in the backup).
MERGE=true
# Skip the interactive confirmation prompt that --no-merge triggers in
# backup-restore.sh (useful for CI/automation).
ASSUME_YES=false

while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
case "$1" in
Expand All @@ -51,6 +62,9 @@ while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
--skip-account) SKIP_ACCOUNT=true; shift ;;
--skip-workspace) SKIP_WORKSPACE=true; shift ;;
--date) DATE="$2"; shift 2 ;;
--merge) MERGE=true; shift ;;
--no-merge) MERGE=false; shift ;;
-y|--yes) ASSUME_YES=true; shift ;;
*)
echo -e "\033[1;31mUnknown option: $1\033[0m"
exit 1
Expand All @@ -74,7 +88,10 @@ if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then
echo " --skip-account Account already exists, don't create it"
echo " --skip-workspace Workspace already exists, don't create/assign it"
echo " --date <ms> Backup snapshot timestamp (default: latest)"
echo " -- <args> Extra args passed to backup-restore.sh tool call (e.g. --merge)"
echo " --merge Keep existing data, only add/update from backup (default)"
echo " --no-merge Destructive: make the workspace match the backup exactly"
echo " -y, --yes Skip the confirmation prompt triggered by --no-merge"
echo " -- <args> Extra args passed to backup-restore.sh tool call (e.g. --recheck)"
exit 1
fi

Expand All @@ -98,6 +115,7 @@ echo " Workspace: $WORKSPACE"
echo " Admin: $EMAIL ($FIRST $LAST)"
echo " Account: $([ "$SKIP_ACCOUNT" == true ] && echo 'skip (exists)' || echo 'create')"
echo " Workspace: $([ "$SKIP_WORKSPACE" == true ] && echo 'skip (exists)' || echo 'create + assign')"
echo " Merge: $([ "$MERGE" == true ] && echo 'yes (existing data preserved)' || echo 'no (destructive, matches backup exactly)')"
echo ""

# 1. Create the admin account (tolerate "already exists")
Expand Down Expand Up @@ -132,6 +150,8 @@ fi
echo -e "\n\033[1;34m[4/4] Restoring backup into $WORKSPACE...\033[0m"
RESTORE_CMD=(./backup-restore.sh "$BACKUP_DIR" "$WORKSPACE")
[ -n "$DATE" ] && RESTORE_CMD+=("$DATE")
[ "$MERGE" == true ] && RESTORE_CMD+=(--merge) || RESTORE_CMD+=(--no-merge)
[ "$ASSUME_YES" == true ] && RESTORE_CMD+=(-y)
[ ${#EXTRA_ARGS[@]} -gt 0 ] && RESTORE_CMD+=(-- "${EXTRA_ARGS[@]}")
"${RESTORE_CMD[@]}"

Expand Down
Loading