Merge pull request #1190 from tesshuflower/restic_restore_nonexist_path

Restic restore nonexist path, ignore lost+found on backup empty dir check

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [unreleased]
+
+### Fixed
+
+- Allow restic restore from empty or non-initialized path
+- Ignore lost+found on restic backup when checking for empty source volume
+
 ## [0.9.0]
 
 ### Changed
@@ -282,6 +289,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support for rsync & rclone replication
 - Helm chart to deploy operator
 
+[Unreleased]: https://github.com/backube/volsync/compare/release-0.9...HEAD
 [0.9.0]: https://github.com/backube/volsync/compare/release-0.8...v0.9.0
 [0.8.1]: https://github.com/backube/volsync/compare/release-0.8.0...v0.8.1
 [0.8.0]: https://github.com/backube/volsync/compare/release-0.7...v0.8.0

controllers/mover/restic/logfilter.go

@@ -24,6 +24,10 @@ import (
 var resticRegex = regexp.MustCompile(
 	`^\s*([pP]rocessed)\s.+([fF]iles)|` +
 		`^\s*([sS]napshot)\s.+([sS]aved)|` +
+		`^\s*(No eligible)|` +
+		`(No data)|` +
+		`(Directory is empty)|` +
+		`^\s*([cC]reated)|` +
 		`^\s*([rR]epository)\s.+([oO]pened)|` +
 		`^\s*([rR]estoring)|` +
 		`^\s*([nN]o parent snapshot)|` +

controllers/mover/restic/logfilter_test.go

@@ -66,7 +66,9 @@ Restic completed in 18s
 === Done ===`
 
 			// nolint:lll
-			expectedFilteredResticSourceLogSuccessful := `repository f5bccd54 opened (repository version 2) successfully, password is correct
+			expectedFilteredResticSourceLogSuccessful := `created restic repository f5bccd54c8 at s3:http://minio-api-minio.apps.app-aws-411ga-sno-net2-zp5jq.dev06.red-chesterfield.com/ttest-restic-new
+repository f5bccd54 opened (repository version 2) successfully, password is correct
+created new cache in /home/testuser/DEVFEDORA/volsync/RESTICTESTS/CACHE
 no parent snapshot found, will read all files
 Added to the repository: 12.941 MiB (12.529 MiB stored)
 processed 25 files, 36.658 MiB in 0:12
@@ -81,6 +83,28 @@ Restic completed in 18s`
 			Expect(filteredLines).To(Equal(expectedFilteredResticSourceLogSuccessful))
 		})
 
+		It("Should filter the logs from a successful replication source of an empty dir (skip restic backup)", func() {
+			// Sample backup log for restic mover where data is empty
+			// nolint:lll
+			resticSourceLogSuccessful := `Starting container
+VolSync restic container version: v0.9.1+aa9ab46-dirty
+backup
+restic 0.16.4 compiled with go1.21.8 on linux/amd64
+Testing mandatory env variables
+== Checking directory for content ===
+== Directory is empty skipping backup ===`
+
+			// nolint:lll
+			expectedFilteredResticSourceLogSuccessful := `== Directory is empty skipping backup ===`
+
+			reader := strings.NewReader(resticSourceLogSuccessful)
+			filteredLines, err := utils.FilterLogs(reader, restic.LogLineFilterSuccess)
+			Expect(err).NotTo(HaveOccurred())
+
+			logger.Info("Logs after filter", "filteredLines", filteredLines)
+			Expect(filteredLines).To(Equal(expectedFilteredResticSourceLogSuccessful))
+		})
+
 		It("Should filter the logs from a replication source (restic backup) that performed an unlock", func() {
 			// Sample backup log for restic mover
 			resticSourceLog := `Starting container
@@ -199,4 +223,41 @@ Restic completed in 9s`
 			Expect(filteredLines).To(Equal(expectedFilteredResticDestlogSuccessful))
 		})
 	})
+
+	Context("Restic dest mover logs - empty repo/path (not initialized previously)", func() {
+		// Sample restore log for restic mover
+		// nolint:lll
+		resticDestlogSuccessful := `Starting container
+VolSync restic container version: v0.9.0+169bc5b-dirty
+restore
+restic 0.16.4 compiled with go1.21.8 on linux/amd64
+Testing mandatory env variables
+=== Check for dir initialized ===
+=== Initialize Dir ===
+created restic repository 374c6313cb at s3:http://minio.minio.svc.cluster.local:9000/resticbucket1b
+
+Please note that knowledge of your password is required to access
+the repository. Losing your password means that your data is
+irrecoverably lost.
+=== Starting restore ===
+No eligible snapshots found
+=== No data will be restored ===
+Restic completed in 4s
+=== Done ===`
+
+		// nolint:lll
+		expectedFilteredResticDestlogSuccessful := `created restic repository 374c6313cb at s3:http://minio.minio.svc.cluster.local:9000/resticbucket1b
+No eligible snapshots found
+=== No data will be restored ===
+Restic completed in 4s`
+
+		It("Should filter the logs from a successful replication dest (restic restore)", func() {
+			reader := strings.NewReader(resticDestlogSuccessful)
+			filteredLines, err := utils.FilterLogs(reader, restic.LogLineFilterSuccess)
+			Expect(err).NotTo(HaveOccurred())
+
+			logger.Info("Logs after filter", "filteredLines", filteredLines)
+			Expect(filteredLines).To(Equal(expectedFilteredResticDestlogSuccessful))
+		})
+	})
 })

mover-restic/entry.sh

@@ -40,7 +40,7 @@ function check_var_defined {
 
 function check_contents {
     echo "== Checking directory for content ==="
-    DIR_CONTENTS="$(ls -A "${DATA_DIR}")"
+    DIR_CONTENTS="$(ls -A "${DATA_DIR}" --ignore="lost+found")"
     if [ -z "${DIR_CONTENTS}" ]; then
         echo "== Directory is empty skipping backup ==="
         exit 0
@@ -49,13 +49,16 @@ function check_contents {
 
 # Ensure the repo has been initialized
 function ensure_initialized {
-    echo "== Initialize Dir ======="
+    echo "=== Check for dir initialized ==="
     # Try a restic command and capture the rc & output
     outfile=$(mktemp -q)
     if ! "${RESTIC[@]}" snapshots 2>"$outfile"; then
         output=$(<"$outfile")
         # Match against error string for uninitialized repo
+        # This string also appears when credentials are incorrect (in which case
+        # the following cmd `restic init` will also fail)
         if [[ $output =~ .*(Is there a repository at the following location).* ]]; then
+            echo "=== Initialize Dir ==="
             "${RESTIC[@]}" init
         else
             cat "$outfile"
@@ -252,6 +255,7 @@ function do_restore {
     snapshot_id=$(select_restic_snapshot_to_restore)
     if [[ -z ${snapshot_id} ]]; then
         echo "No eligible snapshots found"
+        echo "=== No data will be restored ==="
     else
     pushd "${DATA_DIR}"
         echo "Selected restic snapshot with id: ${snapshot_id}"
@@ -286,6 +290,7 @@ for op in "$@"; do
             do_prune
             ;;
         "restore")
+            ensure_initialized
             do_restore
             sync
             ;;