From 5046b77dc5ed7ccc6b952c12286fe1f7d5f915f4 Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:57:48 -0700 Subject: [PATCH 1/9] Add BACKUP_ON_STARTUP option --- scripts/opt/backup-loop.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/opt/backup-loop.sh b/scripts/opt/backup-loop.sh index 4d53cde..02f4d06 100644 --- a/scripts/opt/backup-loop.sh +++ b/scripts/opt/backup-loop.sh @@ -624,6 +624,9 @@ while true; do log ERROR "Unable to turn saving off. Is the server running?" exit 1 fi + elif [[ $first_run == TRUE && ${BACKUP_ON_STARTUP^^} = FALSE ]]; then + log INFO "Skipping backup on startup" + first_run=false else # paused log INFO "Server is paused, proceeding with backup" From bb8cc2cff72d8c39048ed583a7a1237919663dd7 Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Sat, 3 Aug 2024 14:54:57 -0700 Subject: [PATCH 2/9] Move conditional --- scripts/opt/backup-loop.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/opt/backup-loop.sh b/scripts/opt/backup-loop.sh index 02f4d06..4d53cde 100644 --- a/scripts/opt/backup-loop.sh +++ b/scripts/opt/backup-loop.sh @@ -624,9 +624,6 @@ while true; do log ERROR "Unable to turn saving off. Is the server running?" exit 1 fi - elif [[ $first_run == TRUE && ${BACKUP_ON_STARTUP^^} = FALSE ]]; then - log INFO "Skipping backup on startup" - first_run=false else # paused log INFO "Server is paused, proceeding with backup" From e32386f282c4c7454ca4f7c7031dccada4480419 Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Sun, 4 Aug 2024 20:40:05 -0700 Subject: [PATCH 3/9] Fix BACKUP_ON_STARTUP with ONE_SHOT=true --- tests/skip-on-startup/test.skip-on-startup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/skip-on-startup/test.skip-on-startup.sh b/tests/skip-on-startup/test.skip-on-startup.sh index 96c46cd..542dc60 100755 --- a/tests/skip-on-startup/test.skip-on-startup.sh +++ b/tests/skip-on-startup/test.skip-on-startup.sh @@ -84,4 +84,3 @@ run_test2 run_test3 exit $overall_status - From 87fc782537cb11d87144bbf157b63ea1b0511b3f Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Tue, 6 Aug 2024 22:53:21 -0700 Subject: [PATCH 4/9] Allow PRESERVE_MANUAL_BACKUPS --- README.md | 1 + scripts/opt/backup-loop.sh | 13 ++- .../docker-compose.override.yml | 4 + .../docker-compose.yml | 35 ++++++++ .../test.preserve-manual-backups.sh | 87 +++++++++++++++++++ 5 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 tests/preserve-manual-backups/docker-compose.override.yml create mode 100644 tests/preserve-manual-backups/docker-compose.yml create mode 100755 tests/preserve-manual-backups/test.preserve-manual-backups.sh diff --git a/README.md b/README.md index 3fcff17..8efccca 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Provides a side-car container to back up [itzg/minecraft-server](https://github. - `BACKUP_ON_STARTUP`=true : Set to false to skip first backup on startup. - `PAUSE_IF_NO_PLAYERS`=false - `PLAYERS_ONLINE_CHECK_INTERVAL`=5m +- `PRESERVE_MANUAL_BACKUPS`=false : Adds a "-preserve" suffix to [on-demand backups](#on-demand-backups) which will be ignored by prune steps - `PRUNE_BACKUPS_DAYS`=7 - `PRUNE_BACKUPS_COUNT`= -disabled unless set (only works with tar/rsync) - `PRUNE_RESTIC_RETENTION`=--keep-within 7d diff --git a/scripts/opt/backup-loop.sh b/scripts/opt/backup-loop.sh index 4d53cde..18af3bd 100644 --- a/scripts/opt/backup-loop.sh +++ b/scripts/opt/backup-loop.sh @@ -77,6 +77,11 @@ is_one_shot() { fi } +preserve_suffix="" +if is_one_shot; then + preserve_suffix="-preserve" +fi + is_paused() { [[ -e "${SRC_DIR}/.paused" ]] } @@ -229,11 +234,11 @@ tar() { readarray -td, includes_patterns < <(printf '%s' "${INCLUDES:-.}") _find_old_backups() { - find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -mtime "+${PRUNE_BACKUPS_DAYS}" "${@}" + find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -not -name "*${preserve_suffix}*" -mtime "+${PRUNE_BACKUPS_DAYS}" "${@}" } _find_extra_backups() { - find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -exec ls -NtA {} \+ | \ + find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -not -name "*${preserve_suffix}*" -exec ls -NtA {} \+ | \ tail -n +$((PRUNE_BACKUPS_COUNT + 1)) } @@ -263,7 +268,7 @@ tar() { } backup() { ts=$(date +"%Y%m%d-%H%M%S") - outFile="${DEST_DIR}/${BACKUP_NAME}-${ts}.${backup_extension}" + outFile="${DEST_DIR}/${BACKUP_NAME}-${ts}${preserve_suffix}.${backup_extension}" log INFO "Backing up content in ${SRC_DIR} to ${outFile}" command tar "${excludes[@]}" "${tar_parameters[@]}" -cf "${outFile}" -C "${SRC_DIR}" "${includes_patterns[@]}" || exitCode=$? if [ ${exitCode:-0} -eq 0 ]; then @@ -275,7 +280,7 @@ tar() { exit 1 fi if [ "${LINK_LATEST^^}" == "TRUE" ]; then - ln -sf "${BACKUP_NAME}-${ts}.${backup_extension}" "${DEST_DIR}/latest.${backup_extension}" + ln -sf "${BACKUP_NAME}-${ts}${preserve_suffix}.${backup_extension}" "${DEST_DIR}/latest.${backup_extension}" fi } prune() { diff --git a/tests/preserve-manual-backups/docker-compose.override.yml b/tests/preserve-manual-backups/docker-compose.override.yml new file mode 100644 index 0000000..1e755ef --- /dev/null +++ b/tests/preserve-manual-backups/docker-compose.override.yml @@ -0,0 +1,4 @@ +services: + backup: + environment: + PRESERVE_MANUAL_BACKUPS: true diff --git a/tests/preserve-manual-backups/docker-compose.yml b/tests/preserve-manual-backups/docker-compose.yml new file mode 100644 index 0000000..a8fa3f1 --- /dev/null +++ b/tests/preserve-manual-backups/docker-compose.yml @@ -0,0 +1,35 @@ +services: + mc: + image: itzg/minecraft-server + restart: always + tty: true + stdin_open: true + ports: + - "25565:25565" + environment: + EULA: "TRUE" + ENABLE_AUTOPAUSE: "TRUE" + AUTOPAUSE_TIMEOUT_EST: 30 + volumes: + - "./data:/data" + + backup: + restart: "no" + build: ../../ + depends_on: + mc: + condition: service_healthy + environment: + BACKUP_INTERVAL: "1h" + BACKUP_ON_STARTUP: false # isolate to only ONE_SHOT backups + PRUNE_BACKUPS_DAYS: 1 # create a file 2 days old to get pruned in test + RCON_HOST: mc + INITIAL_DELAY: 0 + PAUSE_IF_NO_PLAYERS: true + deploy: + resources: + limits: + memory: 2G + volumes: + - "./data:/data:ro" + - "./backups:/backups" diff --git a/tests/preserve-manual-backups/test.preserve-manual-backups.sh b/tests/preserve-manual-backups/test.preserve-manual-backups.sh new file mode 100755 index 0000000..4026135 --- /dev/null +++ b/tests/preserve-manual-backups/test.preserve-manual-backups.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +mkdir -p ./backups +mkdir -p ./data + +cleanup_backups() { + rm -rf ./backups/* +} + +cleanup_all() { + docker compose down + cleanup_backups + rm -rf ./data/* +} + +## Clean up upon failure or reaching the end +trap cleanup_all EXIT + +setup() { + # Set inital exit state to PASS (0) + overall_status=0 + + # Build from current filesystem + echo "Building..." + docker compose build > /dev/null + + echo "Starting server..." + rm -rf ./data/* + docker compose up mc -d > /dev/null +} + +old_timestamp=$(TZ=UTC+96 date +%Y%m%d%H%M) # Two days old + +run_test1(){ + echo -e "\nTest 1: Ensure default behavior, no suffix" + cleanup_backups + + docker compose run --build backup now + preserved_backup_count=$(find backups/ -name "*.tgz" -name "*preserve*" | wc -l) + not_preserved_backup_count=$(find backups/ -name "*.tgz" -not -name "*preserve*" | wc -l) + + echo "Preserved backups: ${preserved_backup_count}" + echo "Not-Preserved backups: ${not_preserved_backup_count}" + + # Ensure typical backup does not have "-preserve" suffix + if [ 1 -eq "$not_preserved_backup_count" ]; then + echo "PASS" + else + echo "FAIL" + overall_status=1 + fi +} + +run_test2(){ + echo -e "\nTest 1: Ensure added suffix and preserved are not pruned" + cleanup_backups + + # Two old backups + touch -t "$old_timestamp" "./backups/fake-backup-preserve.tgz" + touch -t "$old_timestamp" "./backups/fake-backup.tgz" + + # Plus current preserved backup + docker compose -f docker-compose.yml -f docker-compose.override.yml run --build backup now > /dev/null + + preserved_backup_count=$(find backups/ -name "*.tgz" -name "*preserve*" | wc -l) + not_preserved_backup_count=$(find backups/ -name "*.tgz" -not -name "*preserve*" | wc -l) + + echo "Preserved backups: ${preserved_backup_count}" + echo "Not-Preserved backups: ${not_preserved_backup_count}" + + # Ensure there's + # 2 preserved (1 old + 1 new) + # 0 not preserved (1 pruned) + if [[ 2 -eq "$preserved_backup_count" && 0 -eq "$not_preserved_backup_count" ]]; then + echo "PASS" + else + echo "FAIL" + overall_status=1 + fi +} + +setup +run_test1 +run_test2 + +exit $overall_status + From 29561b010992729a76f895ee607fdecd7651cca5 Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Sat, 10 Aug 2024 19:37:12 -0700 Subject: [PATCH 5/9] Allow new tests to be run from root dire --- scripts/opt/backup-loop.sh | 8 +++++--- .../test.preserve-manual-backups.sh | 12 ++++++++++-- tests/skip-on-startup/test.skip-on-startup.sh | 6 +++++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/opt/backup-loop.sh b/scripts/opt/backup-loop.sh index 18af3bd..b01b15a 100644 --- a/scripts/opt/backup-loop.sh +++ b/scripts/opt/backup-loop.sh @@ -22,6 +22,7 @@ fi : "${BACKUP_ON_STARTUP:=true}" : "${PAUSE_IF_NO_PLAYERS:=false}" : "${PLAYERS_ONLINE_CHECK_INTERVAL:=5m}" +: "${PRESERVE_MANUAL_BACKUPS:=false}" : "${BACKUP_METHOD:=tar}" # currently one of tar, restic, rsync : "${TAR_COMPRESS_METHOD:=gzip}" # bzip2 gzip zstd : "${ZSTD_PARAMETERS:=-3 --long=25 --single-thread}" @@ -77,9 +78,10 @@ is_one_shot() { fi } -preserve_suffix="" -if is_one_shot; then - preserve_suffix="-preserve" +if is_one_shot && [[ "${PRESERVE_MANUAL_BACKUPS^^}" = TRUE ]]; then + preserve_suffix="-preserve" +else + preserve_suffix="" fi is_paused() { diff --git a/tests/preserve-manual-backups/test.preserve-manual-backups.sh b/tests/preserve-manual-backups/test.preserve-manual-backups.sh index 4026135..86e41a5 100755 --- a/tests/preserve-manual-backups/test.preserve-manual-backups.sh +++ b/tests/preserve-manual-backups/test.preserve-manual-backups.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +set -x +WORKDIR="$(realpath "$(dirname "${0}")")" +cd "${WORKDIR}/" || exit + mkdir -p ./backups mkdir -p ./data @@ -35,10 +39,14 @@ run_test1(){ echo -e "\nTest 1: Ensure default behavior, no suffix" cleanup_backups - docker compose run --build backup now + docker compose -f docker-compose.yml config + + docker compose -f docker-compose.yml run --rm --build backup now preserved_backup_count=$(find backups/ -name "*.tgz" -name "*preserve*" | wc -l) not_preserved_backup_count=$(find backups/ -name "*.tgz" -not -name "*preserve*" | wc -l) + tree backups + echo "Preserved backups: ${preserved_backup_count}" echo "Not-Preserved backups: ${not_preserved_backup_count}" @@ -60,7 +68,7 @@ run_test2(){ touch -t "$old_timestamp" "./backups/fake-backup.tgz" # Plus current preserved backup - docker compose -f docker-compose.yml -f docker-compose.override.yml run --build backup now > /dev/null + docker compose -f docker-compose.yml -f docker-compose.override.yml run --rm --build backup now > /dev/null preserved_backup_count=$(find backups/ -name "*.tgz" -name "*preserve*" | wc -l) not_preserved_backup_count=$(find backups/ -name "*.tgz" -not -name "*preserve*" | wc -l) diff --git a/tests/skip-on-startup/test.skip-on-startup.sh b/tests/skip-on-startup/test.skip-on-startup.sh index 542dc60..9d39b20 100755 --- a/tests/skip-on-startup/test.skip-on-startup.sh +++ b/tests/skip-on-startup/test.skip-on-startup.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash +set -x +WORKDIR="$(realpath "$(dirname "${0}")")" +cd "${WORKDIR}/" || exit + mkdir -p ./backups mkdir -p ./data get_backup_count() { - backup_count=$(ls -1 backups | wc -l) + backup_count=$(find backups/* -prune | wc -l) echo "Output: ${backup_count} backups" } From add6e3243a95c01eb46e9b5e983389acafe5cc6c Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Sat, 10 Aug 2024 19:39:40 -0700 Subject: [PATCH 6/9] Add tests to github actions --- .github/workflows/verify.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 55e57a9..d4cd3cd 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -66,3 +66,9 @@ jobs: INITIAL_DELAY: 0s RESTIC_PASSWORD: 1234 run: ./tests/test.simple.restic.sh + + - name: Test skipt on startup + run: ./tests/skip-on-startup/test.skip-on-startup.sh + + - name: Test preserve manual backups + run: ./tests/preserve-manual-backups/test.preserve-manual-backups.sh From 11ec03d33fe14ac062728adbf59f9d2490c0f0db Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Sat, 10 Aug 2024 19:46:40 -0700 Subject: [PATCH 7/9] Correct bug with dynamic prefix --- .github/workflows/verify.yml | 2 +- scripts/opt/backup-loop.sh | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index d4cd3cd..5fc09b1 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -67,7 +67,7 @@ jobs: RESTIC_PASSWORD: 1234 run: ./tests/test.simple.restic.sh - - name: Test skipt on startup + - name: Test skip on startup run: ./tests/skip-on-startup/test.skip-on-startup.sh - name: Test preserve manual backups diff --git a/scripts/opt/backup-loop.sh b/scripts/opt/backup-loop.sh index b01b15a..52314ed 100644 --- a/scripts/opt/backup-loop.sh +++ b/scripts/opt/backup-loop.sh @@ -78,10 +78,12 @@ is_one_shot() { fi } +PRESERVE_SUFFIX="-preserve" + if is_one_shot && [[ "${PRESERVE_MANUAL_BACKUPS^^}" = TRUE ]]; then - preserve_suffix="-preserve" + suffix="-preserve" else - preserve_suffix="" + suffix="" fi is_paused() { @@ -236,11 +238,11 @@ tar() { readarray -td, includes_patterns < <(printf '%s' "${INCLUDES:-.}") _find_old_backups() { - find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -not -name "*${preserve_suffix}*" -mtime "+${PRUNE_BACKUPS_DAYS}" "${@}" + find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -not -name "*${PRESERVE_SUFFIX}*" -mtime "+${PRUNE_BACKUPS_DAYS}" "${@}" } _find_extra_backups() { - find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -not -name "*${preserve_suffix}*" -exec ls -NtA {} \+ | \ + find "${DEST_DIR}" -maxdepth 1 -name "*.${backup_extension}" -not -name "*${PRESERVE_SUFFIX}*" -exec ls -NtA {} \+ | \ tail -n +$((PRUNE_BACKUPS_COUNT + 1)) } @@ -270,7 +272,7 @@ tar() { } backup() { ts=$(date +"%Y%m%d-%H%M%S") - outFile="${DEST_DIR}/${BACKUP_NAME}-${ts}${preserve_suffix}.${backup_extension}" + outFile="${DEST_DIR}/${BACKUP_NAME}-${ts}${suffix}.${backup_extension}" log INFO "Backing up content in ${SRC_DIR} to ${outFile}" command tar "${excludes[@]}" "${tar_parameters[@]}" -cf "${outFile}" -C "${SRC_DIR}" "${includes_patterns[@]}" || exitCode=$? if [ ${exitCode:-0} -eq 0 ]; then @@ -282,7 +284,7 @@ tar() { exit 1 fi if [ "${LINK_LATEST^^}" == "TRUE" ]; then - ln -sf "${BACKUP_NAME}-${ts}${preserve_suffix}.${backup_extension}" "${DEST_DIR}/latest.${backup_extension}" + ln -sf "${BACKUP_NAME}-${ts}${suffix}.${backup_extension}" "${DEST_DIR}/latest.${backup_extension}" fi } prune() { From aa3f9b10e8f0df07dd8db98e7e7dc6189d114427 Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Sat, 10 Aug 2024 20:10:08 -0700 Subject: [PATCH 8/9] Cleanup --- .../preserve-manual-backups/test.preserve-manual-backups.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/preserve-manual-backups/test.preserve-manual-backups.sh b/tests/preserve-manual-backups/test.preserve-manual-backups.sh index 86e41a5..6d93f81 100755 --- a/tests/preserve-manual-backups/test.preserve-manual-backups.sh +++ b/tests/preserve-manual-backups/test.preserve-manual-backups.sh @@ -60,7 +60,7 @@ run_test1(){ } run_test2(){ - echo -e "\nTest 1: Ensure added suffix and preserved are not pruned" + echo -e "\nTest 2: Ensure added suffix and preserved are not pruned" cleanup_backups # Two old backups @@ -73,6 +73,8 @@ run_test2(){ preserved_backup_count=$(find backups/ -name "*.tgz" -name "*preserve*" | wc -l) not_preserved_backup_count=$(find backups/ -name "*.tgz" -not -name "*preserve*" | wc -l) + tree backups + echo "Preserved backups: ${preserved_backup_count}" echo "Not-Preserved backups: ${not_preserved_backup_count}" @@ -81,7 +83,7 @@ run_test2(){ # 0 not preserved (1 pruned) if [[ 2 -eq "$preserved_backup_count" && 0 -eq "$not_preserved_backup_count" ]]; then echo "PASS" - else + else echo "FAIL" overall_status=1 fi From 0eddc1a830af9b8d4fffec999d37a1dd7e88866d Mon Sep 17 00:00:00 2001 From: Justin Wyne <1986068+wyne@users.noreply.github.com> Date: Sun, 25 Aug 2024 18:31:50 -0700 Subject: [PATCH 9/9] Actions permissions --- .github/workflows/verify.yml | 12 +++++++----- tests/test.simple.tar.sh | 5 +++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 5fc09b1..06f81da 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -17,6 +17,8 @@ jobs: EXCLUDES: '*.jar,exclude_dir' RCON_PATH: /usr/bin/rcon-cli DEBUG: true + permissions: + contents: write services: registry: image: registry:2 @@ -26,11 +28,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 -# Script needs to be cleaned up a lot for shellcheck'ing -# - name: ShellCheck -# uses: ludeeus/action-shellcheck@1.1.0 -# with: -# ignore: tests + # Script needs to be cleaned up a lot for shellcheck'ing + # - name: ShellCheck + # uses: ludeeus/action-shellcheck@1.1.0 + # with: + # ignore: tests - name: Hadolint Action uses: hadolint/hadolint-action@v3.1.0 diff --git a/tests/test.simple.tar.sh b/tests/test.simple.tar.sh index 9fac1d9..28311cd 100755 --- a/tests/test.simple.tar.sh +++ b/tests/test.simple.tar.sh @@ -3,7 +3,7 @@ set -euo pipefail set -x -WORKDIR="$(readlink -m "$(dirname "${0}")")" +WORKDIR="$(realpath "$(dirname "${0}")")" cd "${WORKDIR}/.." @@ -34,7 +34,8 @@ export RCON_PATH export PRUNE_BACKUPS_DAYS mkdir "${EXTRACT_DIR}" -touch -d "$(( PRUNE_BACKUPS_DAYS + 2 )) days ago" "${LOCAL_DEST_DIR}/fake_backup_that_should_be_deleted.tgz" +old_timestamp=$(TZ=UTC+96 date +%Y%m%d%H%M) # Two days old +touch -t "$old_timestamp" "${LOCAL_DEST_DIR}/fake_backup_that_should_be_deleted.tgz" ls -al "${LOCAL_DEST_DIR}" timeout 50 docker run --rm \