diff --git a/backup-create.sh b/backup-create.sh new file mode 100755 index 0000000..467d94e --- /dev/null +++ b/backup-create.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# Take a backup of a workspace in this local Huly deployment. +# Connection parameters are read from huly_v7.conf (same file docker compose uses). +# +# Backups are incremental: each run only downloads and stores documents that +# are new or changed since the last backup snapshot in (based on +# per-domain hashes). Re-running this against the same directory is cheap and +# safe - it just adds another snapshot on top of what's already there. +# +# Usage: +# ./backup-create.sh [options] [-- ] +# +# Options: +# --full Force a full recheck: compare every document instead of relying on +# stored hashes (slower, use if you suspect the backup is out of sync) +# --full-verify Full verification: download and diff every document against the +# server, not just changed ones (slowest, most thorough) +# --force Force backup even if no transactions changed since last run +# --include ; separated list of domains to include (default: all) +# --skip ; separated list of domains to skip +# --blob-limit Skip blobs larger than this size in MB (default: 5) +# --content-types ; separated content type prefixes to skip downloading (e.g. video/;audio/) +# --timeout Connect timeout in seconds (default: 30) +# --keep-snapshots Compact backup once it has more than N snapshots (default: 14) +# +# Examples: +# ./backup-create.sh ./backups/myws myws +# ./backup-create.sh ./backups/myws myws --full-verify +# ./backup-create.sh ./backups/myws myws --skip fulltext-blob +set -euo pipefail + +CONFIG_FILE="huly_v7.conf" +if [ ! -f "$CONFIG_FILE" ]; then + echo -e "\033[1;31mConfig not found: $CONFIG_FILE. Run ./setup.sh first.\033[0m" + exit 1 +fi +# shellcheck disable=SC1090 +source "$CONFIG_FILE" + +# Args +BACKUP_DIR="${1:-}" +WORKSPACE="${2:-}" +shift $(( $# >= 2 ? 2 : $# )) || true + +FULL=false +FULL_VERIFY=false +FORCE=false +INCLUDE="" +SKIP="" +BLOB_LIMIT="5" +CONTENT_TYPES="" +TIMEOUT="30" +KEEP_SNAPSHOTS="14" +EXTRA_ARGS=() + +while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do + case "$1" in + --full) FULL=true; shift ;; + --full-verify) FULL_VERIFY=true; shift ;; + --force) FORCE=true; shift ;; + --include) INCLUDE="$2"; shift 2 ;; + --skip) SKIP="$2"; shift 2 ;; + --blob-limit) BLOB_LIMIT="$2"; shift 2 ;; + --content-types) CONTENT_TYPES="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --keep-snapshots) KEEP_SNAPSHOTS="$2"; shift 2 ;; + *) + echo -e "\033[1;31mUnknown option: $1\033[0m" + exit 1 + ;; + esac +done +if [ "${1:-}" == "--" ]; then + shift + EXTRA_ARGS=("$@") +fi + +if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then + echo "Usage: $0 [options] [-- ]" + echo "" + echo " Local directory to store/append the backup (created if missing)" + echo " Workspace id/url to back up" + echo " --full Force full recheck of all documents (slower)" + echo " --full-verify Download and diff every document against the server (slowest)" + echo " --force Force backup even if no transactions changed" + echo " --include ; separated list of domains to include (default: all)" + echo " --skip ; separated list of domains to skip" + echo " --blob-limit Skip blobs larger than this size in MB (default: 5)" + echo " --content-types ; separated content type prefixes to skip (e.g. video/;audio/)" + echo " --timeout Connect timeout in seconds (default: 30)" + echo " --keep-snapshots Compact once more than N snapshots exist (default: 14)" + echo " -- Extra args passed to 'tool backup'" + exit 1 +fi + +# Create the backup directory if it doesn't exist yet (first run) +mkdir -p "$BACKUP_DIR" +BACKUP_ABS="$(cd "$BACKUP_DIR" && pwd)" + +# Required config values +: "${SECRET:?SECRET missing in $CONFIG_FILE}" +: "${CR_DB_URL:?CR_DB_URL missing in $CONFIG_FILE}" +: "${HULY_VERSION:?HULY_VERSION missing in $CONFIG_FILE}" +: "${DOCKER_NAME:?DOCKER_NAME missing in $CONFIG_FILE}" + +NETWORK="${DOCKER_NAME}_huly_net" + +echo -e "\033[1;34mCreating backup:\033[0m" +echo " Destination: $BACKUP_ABS" +echo " Workspace: $WORKSPACE" +echo " Full recheck: $FULL" +echo " Full verify: $FULL_VERIFY" +echo " Force: $FORCE" +echo " Include: ${INCLUDE:-*}" +echo " Skip: ${SKIP:-none}" +echo " Blob limit: ${BLOB_LIMIT}mb" +echo " Keep snapshots: $KEEP_SNAPSHOTS" +echo " Network: $NETWORK" +echo " Version: $HULY_VERSION" +echo "" + +# Verify the stack network exists (stack must be up) +if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then + echo -e "\033[1;31mNetwork $NETWORK not found. Start the stack first: docker compose up -d\033[0m" + exit 1 +fi + +CMD=(backup /backup "$WORKSPACE") +[ "$FULL" == true ] && CMD+=(--full) +[ "$FULL_VERIFY" == true ] && CMD+=(--fullVerify) +[ "$FORCE" == true ] && CMD+=(--force) +[ -n "$INCLUDE" ] && CMD+=(--include "$INCLUDE") +[ -n "$SKIP" ] && CMD+=(--skip "$SKIP") +[ -n "$BLOB_LIMIT" ] && CMD+=(--blobLimit "$BLOB_LIMIT") +[ -n "$CONTENT_TYPES" ] && CMD+=(--contentTypes "$CONTENT_TYPES") +[ -n "$TIMEOUT" ] && CMD+=(--timeout "$TIMEOUT") +[ -n "$KEEP_SNAPSHOTS" ] && CMD+=(--keepSnapshots "$KEEP_SNAPSHOTS") +[ ${#EXTRA_ARGS[@]} -gt 0 ] && CMD+=("${EXTRA_ARGS[@]}") + +RUN_TOOL_DOCKER_ARGS="-v ${BACKUP_ABS}:/backup" ./run-tool.sh "${CMD[@]}" + +echo -e "\n\033[1;32mBackup finished.\033[0m" +echo " Stored in: $BACKUP_ABS" diff --git a/backup-restore.sh b/backup-restore.sh index eec5955..7500811 100755 --- a/backup-restore.sh +++ b/backup-restore.sh @@ -3,8 +3,14 @@ # Restore a downloaded backup into this local Huly deployment. # Connection parameters are read from huly_v7.conf (same file docker compose uses). # +# By default this restores in MERGE mode: only new/changed documents from the +# backup are uploaded, and any data already present in the workspace but not +# in the backup is left untouched (nothing is deleted). Use --no-merge to +# restore the destructive way (server is made to match the backup exactly, +# i.e. anything not in the backup gets removed). +# # Usage: -# ./backup-restore.sh [date] [--no-accounts] [--no-upgrade] [-- ] +# ./backup-restore.sh [date] [--no-accounts] [--no-upgrade] [--no-merge] [-y] [-- ] # # Example: # ./backup-restore.sh ./backups/myws myws @@ -29,6 +35,14 @@ RESTORE_ACCOUNTS=true # Upgrade the workspace to the current model version after restore. On by default - # the backup is usually from an older version. Disable with --no-upgrade. UPGRADE=true +# Merge mode: only add/update documents from the backup, never delete anything +# already present in the workspace. On by default so restores are safe to +# re-run and only bring in what's missing. Disable with --no-merge to make +# the workspace match the backup exactly (deletes data not in the backup). +MERGE=true +# Skip the interactive confirmation prompt that --no-merge triggers (useful +# for CI/automation). Has no effect when merge is on. +ASSUME_YES=false shift $(( $# >= 2 ? 2 : $# )) || true @@ -39,6 +53,9 @@ while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do --no-accounts) RESTORE_ACCOUNTS=false; shift ;; --upgrade) UPGRADE=true; shift ;; --no-upgrade) UPGRADE=false; shift ;; + --merge) MERGE=true; shift ;; + --no-merge) MERGE=false; shift ;; + -y|--yes) ASSUME_YES=true; shift ;; *) DATE="$1"; shift ;; esac done @@ -48,14 +65,16 @@ if [ "${1:-}" == "--" ]; then fi if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then - echo "Usage: $0 [date] [--no-accounts] [--no-upgrade] [-- ]" + echo "Usage: $0 [date] [--no-accounts] [--no-upgrade] [--no-merge] [-y] [-- ]" echo "" echo " Local directory with downloaded backup files" echo " Target workspace id/url to restore into" echo " [date] Optional snapshot timestamp (ms). Default: latest" echo " --no-accounts Do not restore person/socialId accounts (default: restore them)" echo " --no-upgrade Do not upgrade the workspace after restore (default: upgrade)" - echo " -- Extra args passed to 'tool backup-restore' (e.g. --merge)" + echo " --no-merge Delete data not present in the backup (default: merge, nothing is deleted)" + echo " -y, --yes Skip the confirmation prompt triggered by --no-merge" + echo " -- Extra args passed to 'tool backup-restore' (e.g. --recheck)" exit 1 fi @@ -81,6 +100,7 @@ echo " Workspace: $WORKSPACE" echo " Date: ${DATE:-latest}" echo " Accounts: ${RESTORE_ACCOUNTS}" echo " Upgrade: ${UPGRADE}" +echo " Merge: ${MERGE} $([ "$MERGE" == true ] && echo '(existing data preserved, only new/changed restored)' || echo '(destructive, matches backup exactly)')" echo " Network: $NETWORK" echo " Version: $HULY_VERSION" echo "" @@ -91,9 +111,25 @@ if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then exit 1 fi +# --no-merge deletes any data in the workspace that isn't present in the +# backup and cannot be undone - require explicit confirmation unless -y/--yes +# was passed (e.g. for scripted/CI use). +if [ "$MERGE" != true ] && [ "$ASSUME_YES" != true ]; then + echo -e "\033[1;31mWARNING: --no-merge will DELETE any data in workspace '$WORKSPACE'\033[0m" + echo -e "\033[1;31mthat is not present in the backup. This cannot be undone.\033[0m" + echo "" + read -r -p "Type the workspace name (${WORKSPACE}) to confirm, anything else to abort: " CONFIRM + if [ "$CONFIRM" != "$WORKSPACE" ]; then + echo "Aborted." + exit 1 + fi + echo "" +fi + CMD=(backup-restore /backup "$WORKSPACE") [ -n "$DATE" ] && CMD+=("$DATE") [ "$RESTORE_ACCOUNTS" == true ] && CMD+=(--accounts) +[ "$MERGE" == true ] && CMD+=(--merge) [ ${#EXTRA_ARGS[@]} -gt 0 ] && CMD+=("${EXTRA_ARGS[@]}") RUN_TOOL_DOCKER_ARGS="-v ${BACKUP_ABS}:/backup" ./run-tool.sh "${CMD[@]}" diff --git a/restore-workspace.sh b/restore-workspace.sh index 67f84ee..9d735cd 100755 --- a/restore-workspace.sh +++ b/restore-workspace.sh @@ -14,11 +14,15 @@ # --skip-account Don't create the account (it already exists) # --skip-workspace Don't create/assign the workspace (it already exists) # --date Backup snapshot timestamp, passed to backup-restore.sh +# --merge Merge mode: keep existing data, only add/update from backup (default) +# --no-merge Destructive mode: make the workspace match the backup exactly +# (prompts for confirmation unless -y/--yes is also given) +# -y, --yes Skip the confirmation prompt triggered by --no-merge # # Examples: # ./restore-workspace.sh ./backups/myws myws # ./restore-workspace.sh ./backups/myws myws -e me@example.com -p secret -# ./restore-workspace.sh ./backups/myws myws --skip-account -- --merge +# ./restore-workspace.sh ./backups/myws myws --skip-account --no-merge set -euo pipefail CONFIG_FILE="huly_v7.conf" @@ -41,6 +45,13 @@ SKIP_ACCOUNT=false SKIP_WORKSPACE=false DATE="" EXTRA_ARGS=() +# Merge mode: only add/update documents from the backup, never delete anything +# already present in the workspace. On by default. Disable with --no-merge to +# make the workspace match the backup exactly (deletes data not in the backup). +MERGE=true +# Skip the interactive confirmation prompt that --no-merge triggers in +# backup-restore.sh (useful for CI/automation). +ASSUME_YES=false while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do case "$1" in @@ -51,6 +62,9 @@ while [ $# -gt 0 ] && [ "${1:-}" != "--" ]; do --skip-account) SKIP_ACCOUNT=true; shift ;; --skip-workspace) SKIP_WORKSPACE=true; shift ;; --date) DATE="$2"; shift 2 ;; + --merge) MERGE=true; shift ;; + --no-merge) MERGE=false; shift ;; + -y|--yes) ASSUME_YES=true; shift ;; *) echo -e "\033[1;31mUnknown option: $1\033[0m" exit 1 @@ -74,7 +88,10 @@ if [ -z "$BACKUP_DIR" ] || [ -z "$WORKSPACE" ]; then echo " --skip-account Account already exists, don't create it" echo " --skip-workspace Workspace already exists, don't create/assign it" echo " --date Backup snapshot timestamp (default: latest)" - echo " -- Extra args passed to backup-restore.sh tool call (e.g. --merge)" + echo " --merge Keep existing data, only add/update from backup (default)" + echo " --no-merge Destructive: make the workspace match the backup exactly" + echo " -y, --yes Skip the confirmation prompt triggered by --no-merge" + echo " -- Extra args passed to backup-restore.sh tool call (e.g. --recheck)" exit 1 fi @@ -98,6 +115,7 @@ echo " Workspace: $WORKSPACE" echo " Admin: $EMAIL ($FIRST $LAST)" echo " Account: $([ "$SKIP_ACCOUNT" == true ] && echo 'skip (exists)' || echo 'create')" echo " Workspace: $([ "$SKIP_WORKSPACE" == true ] && echo 'skip (exists)' || echo 'create + assign')" +echo " Merge: $([ "$MERGE" == true ] && echo 'yes (existing data preserved)' || echo 'no (destructive, matches backup exactly)')" echo "" # 1. Create the admin account (tolerate "already exists") @@ -132,6 +150,8 @@ fi echo -e "\n\033[1;34m[4/4] Restoring backup into $WORKSPACE...\033[0m" RESTORE_CMD=(./backup-restore.sh "$BACKUP_DIR" "$WORKSPACE") [ -n "$DATE" ] && RESTORE_CMD+=("$DATE") +[ "$MERGE" == true ] && RESTORE_CMD+=(--merge) || RESTORE_CMD+=(--no-merge) +[ "$ASSUME_YES" == true ] && RESTORE_CMD+=(-y) [ ${#EXTRA_ARGS[@]} -gt 0 ] && RESTORE_CMD+=(-- "${EXTRA_ARGS[@]}") "${RESTORE_CMD[@]}"