mirror of
https://github.com/hcengineering/huly-selfhost.git
synced 2026-10-01 14:05:07 +02:00
Add script to take backup and support merge in restore (#312)
Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
Artyom Savchenko
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Take a backup of a workspace in this local Huly deployment.
|
||||
# Connection parameters are read from huly_v7.conf (same file docker compose uses).
|
||||
#
|
||||
# Backups are incremental: each run only downloads and stores documents that
|
||||
# are new or changed since the last backup snapshot in <backup-dir> (based on
|
||||
# per-domain hashes). Re-running this against the same directory is cheap and
|
||||
# safe - it just adds another snapshot on top of what's already there.
|
||||
#
|
||||
# Usage:
|
||||
# ./backup-create.sh <backup-dir> <workspace> [options] [-- <extra tool args>]
|
||||
#
|
||||
# Options:
|
||||
# --full Force a full recheck: compare every document instead of relying on
|
||||
# stored hashes (slower, use if you suspect the backup is out of sync)
|
||||
# --full-verify Full verification: download and diff every document against the
|
||||
# server, not just changed ones (slowest, most thorough)
|
||||
# --force Force backup even if no transactions changed since last run
|
||||
# --include <domains> ; separated list of domains to include (default: all)
|
||||
# --skip <domains> ; separated list of domains to skip
|
||||
# --blob-limit <mb> Skip blobs larger than this size in MB (default: 5)
|
||||
# --content-types <ct> ; separated content type prefixes to skip downloading (e.g. video/;audio/)
|
||||
# --timeout <sec> Connect timeout in seconds (default: 30)
|
||||
# --keep-snapshots <n> Compact backup once it has more than N snapshots (default: 14)
|
||||
#
|
||||
# Examples:
|
||||
# ./backup-create.sh ./backups/myws myws
|
||||
# ./backup-create.sh ./backups/myws myws --full-verify
|
||||
# ./backup-create.sh ./backups/myws myws --skip fulltext-blob
|
||||
set -euo pipefail
|
||||
|
||||
CONFIG_FILE="huly_v7.conf"
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
echo -e "\033[1;31mConfig not found: $CONFIG_FILE. Run ./setup.sh first.\033[0m"
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
|
||||
# Args
|
||||
BACKUP_DIR="${1:-}"
|
||||
WORKSPACE="${2:-}"
|
||||
shift $(( $# >= 2 ? 2 : $# )) || true
|
||||
|
||||
FULL=false
|
||||
FULL_VERIFY=false
|
||||
FORCE=false
|
||||
INCLUDE=""
|
||||
SKIP=""
|
||||
BLOB_LIMIT="5"
|
||||
CONTENT_TYPES=""
|
||||
TIMEOUT="30"
|
||||
KEEP_SNAPSHOTS="14"
|
||||
EXTRA_ARGS=()
|
||||
|
||||
while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
|
||||
case "$1" in
|
||||
--full) FULL=true; shift ;;
|
||||
--full-verify) FULL_VERIFY=true; shift ;;
|
||||
--force) FORCE=true; shift ;;
|
||||
--include) INCLUDE="$2"; shift 2 ;;
|
||||
--skip) SKIP="$2"; shift 2 ;;
|
||||
--blob-limit) BLOB_LIMIT="$2"; shift 2 ;;
|
||||
--content-types) CONTENT_TYPES="$2"; shift 2 ;;
|
||||
--timeout) TIMEOUT="$2"; shift 2 ;;
|
||||
--keep-snapshots) KEEP_SNAPSHOTS="$2"; shift 2 ;;
|
||||
*)
|
||||
echo -e "\033[1;31mUnknown option: $1\033[0m"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [ "${1:-}" == "--" ]; then
|
||||
shift
|
||||
EXTRA_ARGS=("$@")
|
||||
fi
|
||||
|
||||
if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then
|
||||
echo "Usage: $0 <backup-dir> <workspace> [options] [-- <extra tool args>]"
|
||||
echo ""
|
||||
echo " <backup-dir> Local directory to store/append the backup (created if missing)"
|
||||
echo " <workspace> Workspace id/url to back up"
|
||||
echo " --full Force full recheck of all documents (slower)"
|
||||
echo " --full-verify Download and diff every document against the server (slowest)"
|
||||
echo " --force Force backup even if no transactions changed"
|
||||
echo " --include <d> ; separated list of domains to include (default: all)"
|
||||
echo " --skip <d> ; separated list of domains to skip"
|
||||
echo " --blob-limit <mb> Skip blobs larger than this size in MB (default: 5)"
|
||||
echo " --content-types <t> ; separated content type prefixes to skip (e.g. video/;audio/)"
|
||||
echo " --timeout <sec> Connect timeout in seconds (default: 30)"
|
||||
echo " --keep-snapshots <n> Compact once more than N snapshots exist (default: 14)"
|
||||
echo " -- <args> Extra args passed to 'tool backup'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create the backup directory if it doesn't exist yet (first run)
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
BACKUP_ABS="$(cd "$BACKUP_DIR" && pwd)"
|
||||
|
||||
# Required config values
|
||||
: "${SECRET:?SECRET missing in $CONFIG_FILE}"
|
||||
: "${CR_DB_URL:?CR_DB_URL missing in $CONFIG_FILE}"
|
||||
: "${HULY_VERSION:?HULY_VERSION missing in $CONFIG_FILE}"
|
||||
: "${DOCKER_NAME:?DOCKER_NAME missing in $CONFIG_FILE}"
|
||||
|
||||
NETWORK="${DOCKER_NAME}_huly_net"
|
||||
|
||||
echo -e "\033[1;34mCreating backup:\033[0m"
|
||||
echo " Destination: $BACKUP_ABS"
|
||||
echo " Workspace: $WORKSPACE"
|
||||
echo " Full recheck: $FULL"
|
||||
echo " Full verify: $FULL_VERIFY"
|
||||
echo " Force: $FORCE"
|
||||
echo " Include: ${INCLUDE:-*}"
|
||||
echo " Skip: ${SKIP:-none}"
|
||||
echo " Blob limit: ${BLOB_LIMIT}mb"
|
||||
echo " Keep snapshots: $KEEP_SNAPSHOTS"
|
||||
echo " Network: $NETWORK"
|
||||
echo " Version: $HULY_VERSION"
|
||||
echo ""
|
||||
|
||||
# Verify the stack network exists (stack must be up)
|
||||
if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
|
||||
echo -e "\033[1;31mNetwork $NETWORK not found. Start the stack first: docker compose up -d\033[0m"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CMD=(backup /backup "$WORKSPACE")
|
||||
[ "$FULL" == true ] && CMD+=(--full)
|
||||
[ "$FULL_VERIFY" == true ] && CMD+=(--fullVerify)
|
||||
[ "$FORCE" == true ] && CMD+=(--force)
|
||||
[ -n "$INCLUDE" ] && CMD+=(--include "$INCLUDE")
|
||||
[ -n "$SKIP" ] && CMD+=(--skip "$SKIP")
|
||||
[ -n "$BLOB_LIMIT" ] && CMD+=(--blobLimit "$BLOB_LIMIT")
|
||||
[ -n "$CONTENT_TYPES" ] && CMD+=(--contentTypes "$CONTENT_TYPES")
|
||||
[ -n "$TIMEOUT" ] && CMD+=(--timeout "$TIMEOUT")
|
||||
[ -n "$KEEP_SNAPSHOTS" ] && CMD+=(--keepSnapshots "$KEEP_SNAPSHOTS")
|
||||
[ ${#EXTRA_ARGS[@]} -gt 0 ] && CMD+=("${EXTRA_ARGS[@]}")
|
||||
|
||||
RUN_TOOL_DOCKER_ARGS="-v ${BACKUP_ABS}:/backup" ./run-tool.sh "${CMD[@]}"
|
||||
|
||||
echo -e "\n\033[1;32mBackup finished.\033[0m"
|
||||
echo " Stored in: $BACKUP_ABS"
|
||||
+39
-3
@@ -3,8 +3,14 @@
|
||||
# Restore a downloaded backup into this local Huly deployment.
|
||||
# Connection parameters are read from huly_v7.conf (same file docker compose uses).
|
||||
#
|
||||
# By default this restores in MERGE mode: only new/changed documents from the
|
||||
# backup are uploaded, and any data already present in the workspace but not
|
||||
# in the backup is left untouched (nothing is deleted). Use --no-merge to
|
||||
# restore the destructive way (server is made to match the backup exactly,
|
||||
# i.e. anything not in the backup gets removed).
|
||||
#
|
||||
# Usage:
|
||||
# ./backup-restore.sh <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [-- <extra tool args>]
|
||||
# ./backup-restore.sh <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [--no-merge] [-y] [-- <extra tool args>]
|
||||
#
|
||||
# Example:
|
||||
# ./backup-restore.sh ./backups/myws myws
|
||||
@@ -29,6 +35,14 @@ RESTORE_ACCOUNTS=true
|
||||
# Upgrade the workspace to the current model version after restore. On by default -
|
||||
# the backup is usually from an older version. Disable with --no-upgrade.
|
||||
UPGRADE=true
|
||||
# Merge mode: only add/update documents from the backup, never delete anything
|
||||
# already present in the workspace. On by default so restores are safe to
|
||||
# re-run and only bring in what's missing. Disable with --no-merge to make
|
||||
# the workspace match the backup exactly (deletes data not in the backup).
|
||||
MERGE=true
|
||||
# Skip the interactive confirmation prompt that --no-merge triggers (useful
|
||||
# for CI/automation). Has no effect when merge is on.
|
||||
ASSUME_YES=false
|
||||
|
||||
shift $(( $# >= 2 ? 2 : $# )) || true
|
||||
|
||||
@@ -39,6 +53,9 @@ while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
|
||||
--no-accounts) RESTORE_ACCOUNTS=false; shift ;;
|
||||
--upgrade) UPGRADE=true; shift ;;
|
||||
--no-upgrade) UPGRADE=false; shift ;;
|
||||
--merge) MERGE=true; shift ;;
|
||||
--no-merge) MERGE=false; shift ;;
|
||||
-y|--yes) ASSUME_YES=true; shift ;;
|
||||
*) DATE="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
@@ -48,14 +65,16 @@ if [ "${1:-}" == "--" ]; then
|
||||
fi
|
||||
|
||||
if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then
|
||||
echo "Usage: $0 <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [-- <extra tool args>]"
|
||||
echo "Usage: $0 <backup-dir> <workspace> [date] [--no-accounts] [--no-upgrade] [--no-merge] [-y] [-- <extra tool args>]"
|
||||
echo ""
|
||||
echo " <backup-dir> Local directory with downloaded backup files"
|
||||
echo " <workspace> Target workspace id/url to restore into"
|
||||
echo " [date] Optional snapshot timestamp (ms). Default: latest"
|
||||
echo " --no-accounts Do not restore person/socialId accounts (default: restore them)"
|
||||
echo " --no-upgrade Do not upgrade the workspace after restore (default: upgrade)"
|
||||
echo " -- <args> Extra args passed to 'tool backup-restore' (e.g. --merge)"
|
||||
echo " --no-merge Delete data not present in the backup (default: merge, nothing is deleted)"
|
||||
echo " -y, --yes Skip the confirmation prompt triggered by --no-merge"
|
||||
echo " -- <args> Extra args passed to 'tool backup-restore' (e.g. --recheck)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -81,6 +100,7 @@ echo " Workspace: $WORKSPACE"
|
||||
echo " Date: ${DATE:-latest}"
|
||||
echo " Accounts: ${RESTORE_ACCOUNTS}"
|
||||
echo " Upgrade: ${UPGRADE}"
|
||||
echo " Merge: ${MERGE} $([ "$MERGE" == true ] && echo '(existing data preserved, only new/changed restored)' || echo '(destructive, matches backup exactly)')"
|
||||
echo " Network: $NETWORK"
|
||||
echo " Version: $HULY_VERSION"
|
||||
echo ""
|
||||
@@ -91,9 +111,25 @@ if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --no-merge deletes any data in the workspace that isn't present in the
|
||||
# backup and cannot be undone - require explicit confirmation unless -y/--yes
|
||||
# was passed (e.g. for scripted/CI use).
|
||||
if [ "$MERGE" != true ] && [ "$ASSUME_YES" != true ]; then
|
||||
echo -e "\033[1;31mWARNING: --no-merge will DELETE any data in workspace '$WORKSPACE'\033[0m"
|
||||
echo -e "\033[1;31mthat is not present in the backup. This cannot be undone.\033[0m"
|
||||
echo ""
|
||||
read -r -p "Type the workspace name (${WORKSPACE}) to confirm, anything else to abort: " CONFIRM
|
||||
if [ "$CONFIRM" != "$WORKSPACE" ]; then
|
||||
echo "Aborted."
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
CMD=(backup-restore /backup "$WORKSPACE")
|
||||
[ -n "$DATE" ] && CMD+=("$DATE")
|
||||
[ "$RESTORE_ACCOUNTS" == true ] && CMD+=(--accounts)
|
||||
[ "$MERGE" == true ] && CMD+=(--merge)
|
||||
[ ${#EXTRA_ARGS[@]} -gt 0 ] && CMD+=("${EXTRA_ARGS[@]}")
|
||||
|
||||
RUN_TOOL_DOCKER_ARGS="-v ${BACKUP_ABS}:/backup" ./run-tool.sh "${CMD[@]}"
|
||||
|
||||
+22
-2
@@ -14,11 +14,15 @@
|
||||
# --skip-account Don't create the account (it already exists)
|
||||
# --skip-workspace Don't create/assign the workspace (it already exists)
|
||||
# --date <ms> Backup snapshot timestamp, passed to backup-restore.sh
|
||||
# --merge Merge mode: keep existing data, only add/update from backup (default)
|
||||
# --no-merge Destructive mode: make the workspace match the backup exactly
|
||||
# (prompts for confirmation unless -y/--yes is also given)
|
||||
# -y, --yes Skip the confirmation prompt triggered by --no-merge
|
||||
#
|
||||
# Examples:
|
||||
# ./restore-workspace.sh ./backups/myws myws
|
||||
# ./restore-workspace.sh ./backups/myws myws -e me@example.com -p secret
|
||||
# ./restore-workspace.sh ./backups/myws myws --skip-account -- --merge
|
||||
# ./restore-workspace.sh ./backups/myws myws --skip-account --no-merge
|
||||
set -euo pipefail
|
||||
|
||||
CONFIG_FILE="huly_v7.conf"
|
||||
@@ -41,6 +45,13 @@ SKIP_ACCOUNT=false
|
||||
SKIP_WORKSPACE=false
|
||||
DATE=""
|
||||
EXTRA_ARGS=()
|
||||
# Merge mode: only add/update documents from the backup, never delete anything
|
||||
# already present in the workspace. On by default. Disable with --no-merge to
|
||||
# make the workspace match the backup exactly (deletes data not in the backup).
|
||||
MERGE=true
|
||||
# Skip the interactive confirmation prompt that --no-merge triggers in
|
||||
# backup-restore.sh (useful for CI/automation).
|
||||
ASSUME_YES=false
|
||||
|
||||
while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
|
||||
case "$1" in
|
||||
@@ -51,6 +62,9 @@ while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do
|
||||
--skip-account) SKIP_ACCOUNT=true; shift ;;
|
||||
--skip-workspace) SKIP_WORKSPACE=true; shift ;;
|
||||
--date) DATE="$2"; shift 2 ;;
|
||||
--merge) MERGE=true; shift ;;
|
||||
--no-merge) MERGE=false; shift ;;
|
||||
-y|--yes) ASSUME_YES=true; shift ;;
|
||||
*)
|
||||
echo -e "\033[1;31mUnknown option: $1\033[0m"
|
||||
exit 1
|
||||
@@ -74,7 +88,10 @@ if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then
|
||||
echo " --skip-account Account already exists, don't create it"
|
||||
echo " --skip-workspace Workspace already exists, don't create/assign it"
|
||||
echo " --date <ms> Backup snapshot timestamp (default: latest)"
|
||||
echo " -- <args> Extra args passed to backup-restore.sh tool call (e.g. --merge)"
|
||||
echo " --merge Keep existing data, only add/update from backup (default)"
|
||||
echo " --no-merge Destructive: make the workspace match the backup exactly"
|
||||
echo " -y, --yes Skip the confirmation prompt triggered by --no-merge"
|
||||
echo " -- <args> Extra args passed to backup-restore.sh tool call (e.g. --recheck)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -98,6 +115,7 @@ echo " Workspace: $WORKSPACE"
|
||||
echo " Admin: $EMAIL ($FIRST $LAST)"
|
||||
echo " Account: $([ "$SKIP_ACCOUNT" == true ] && echo 'skip (exists)' || echo 'create')"
|
||||
echo " Workspace: $([ "$SKIP_WORKSPACE" == true ] && echo 'skip (exists)' || echo 'create + assign')"
|
||||
echo " Merge: $([ "$MERGE" == true ] && echo 'yes (existing data preserved)' || echo 'no (destructive, matches backup exactly)')"
|
||||
echo ""
|
||||
|
||||
# 1. Create the admin account (tolerate "already exists")
|
||||
@@ -132,6 +150,8 @@ fi
|
||||
echo -e "\n\033[1;34m[4/4] Restoring backup into $WORKSPACE...\033[0m"
|
||||
RESTORE_CMD=(./backup-restore.sh "$BACKUP_DIR" "$WORKSPACE")
|
||||
[ -n "$DATE" ] && RESTORE_CMD+=("$DATE")
|
||||
[ "$MERGE" == true ] && RESTORE_CMD+=(--merge) || RESTORE_CMD+=(--no-merge)
|
||||
[ "$ASSUME_YES" == true ] && RESTORE_CMD+=(-y)
|
||||
[ ${#EXTRA_ARGS[@]} -gt 0 ] && RESTORE_CMD+=(-- "${EXTRA_ARGS[@]}")
|
||||
"${RESTORE_CMD[@]}"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user