Merge pull request #2 from KYVENetwork/feat/backup

Feat/backup

Makefile

@@ -1,4 +1,4 @@
-VERSION := v0.2.1
+VERSION := v0.2.2
 
 ldflags := $(LDFLAGS)
 ldflags += -X main.Version=$(VERSION)

backup/backup.go

@@ -0,0 +1,120 @@
+package backup
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+func ClearBackups(srcPath string, threshold int) error {
+	// Get and sort all created Backups
+	entries, err := os.ReadDir(srcPath)
+	if err != nil {
+		return err
+	}
+
+	backups := []os.DirEntry{}
+	for _, entry := range entries {
+		if entry.IsDir() {
+			// Make sure to only clear timestamped backups
+			if strings.HasPrefix(entry.Name(), "20") && len(entry.Name()) == 15 {
+				backups = append(backups, entry)
+			}
+		}
+	}
+
+	sort.Slice(backups, func(i, j int) bool {
+		return backups[i].Name() < backups[j].Name()
+	})
+
+	if len(backups) > threshold {
+		for {
+			oldestBackup := backups[0].Name()
+			err = os.RemoveAll(filepath.Join(srcPath, oldestBackup))
+			if err != nil {
+				return err
+			}
+
+			backups = backups[1:]
+
+			if len(backups) <= threshold {
+				break
+			}
+		}
+	}
+	return nil
+}
+
+// CompressDirectory compresses a directory using Gzip and creates a .tar.gz file.
+func CompressDirectory(srcPath, compressionType string) error {
+	var cmd *exec.Cmd
+
+	switch compressionType {
+	case "tar.gz":
+		cmd = exec.Command("tar", "-zcvf", filepath.Base(srcPath)+"."+compressionType, filepath.Base(srcPath))
+	case "zip":
+		cmd = exec.Command("zip", "-r", filepath.Base(srcPath)+"."+compressionType, filepath.Base(srcPath))
+	default:
+		return fmt.Errorf("unsupported compression type")
+	}
+
+	cmd.Dir = filepath.Dir(srcPath)
+	cmd.Stderr = os.Stderr
+
+	// Run the command
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	if err := os.RemoveAll(srcPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func CopyDir(srcDir, destDir string) error {
+	// Create the destination directory if it doesn't exist
+	if err := os.MkdirAll(destDir, 0o755); err != nil {
+		return err
+	}
+
+	// Walk through the source directory and copy its contents to the destination
+	return filepath.Walk(srcDir, func(srcPath string, fileInfo os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Construct the corresponding destination path
+		destPath := filepath.Join(destDir, srcPath[len(srcDir):])
+
+		if fileInfo.IsDir() {
+			// Create the destination directory if it doesn't exist
+			return os.MkdirAll(destPath, 0o755)
+		} else {
+			// Open the source file for reading
+			srcFile, err := os.Open(srcPath)
+			if err != nil {
+				return err
+			}
+			defer srcFile.Close()
+
+			// Create the destination file
+			destFile, err := os.Create(destPath)
+			if err != nil {
+				return err
+			}
+			defer destFile.Close()
+
+			// Copy the contents from source to destination
+			if _, err := io.Copy(destFile, srcFile); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}

cmd/supervysor/backup.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/KYVENetwork/supervysor/backup"
+	"github.com/KYVENetwork/supervysor/cmd/supervysor/helpers"
+	"github.com/spf13/cobra"
+)
+
+var (
+	compressionType string
+	destPath        string
+	maxBackups      int
+	srcPath         string
+)
+
+func init() {
+	backupCmd.Flags().StringVar(&srcPath, "src-path", "", "source path of the directory to backup")
+	if err := backupCmd.MarkFlagRequired("src-path"); err != nil {
+		panic(fmt.Errorf("flag 'src-path' should be required: %w", err))
+	}
+
+	backupCmd.Flags().StringVar(&destPath, "dest-path", "", "destination path of the written backup (default '~/.ksync/backups)'")
+
+	backupCmd.Flags().StringVar(&compressionType, "compression", "", "compression type to compress backup directory ['tar.gz', 'zip', '']")
+
+	backupCmd.Flags().IntVar(&maxBackups, "max-backups", 0, "number of kept backups (set 0 to keep all)")
+}
+
+var backupCmd = &cobra.Command{
+	Use:   "backup",
+	Short: "Backup data directory",
+	Run: func(cmd *cobra.Command, args []string) {
+		backupDir, err := helpers.GetBackupDir()
+		if err != nil {
+			logger.Error("failed to get ksync home directory", "err", err)
+			return
+		}
+
+		if destPath == "" {
+			d, err := helpers.CreateDestPath(backupDir)
+			if err != nil {
+				return
+			}
+			destPath = d
+		}
+
+		if err := helpers.ValidatePaths(srcPath, destPath); err != nil {
+			return
+		}
+
+		logger.Info("starting to copy backup", "from", srcPath, "to", destPath)
+
+		if err := backup.CopyDir(srcPath, destPath); err != nil {
+			logger.Error("error copying directory to backup destination", "err", err)
+		}
+
+		logger.Info("directory copied successfully")
+
+		if compressionType != "" {
+			if err := backup.CompressDirectory(destPath, compressionType); err != nil {
+				logger.Error("compression failed", "err", err)
+			}
+
+			logger.Info("compressed backup successfully")
+		}
+
+		if maxBackups > 0 {
+			logger.Info("starting to cleanup backup directory", "path", backupDir)
+			if err := backup.ClearBackups(backupDir, maxBackups); err != nil {
+				logger.Error("clearing backup directory failed", "err", err)
+				return
+			}
+		}
+	},
+}

cmd/supervysor/helpers/helpers.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/spf13/viper"
 
@@ -14,6 +15,18 @@ import (
 	cfg "github.com/tendermint/tendermint/config"
 )
 
+func CreateDestPath(backupDir string) (string, error) {
+	t := time.Now().Format("20060102_150405")
+
+	if err := os.Mkdir(filepath.Join(backupDir, t), 0o755); err != nil {
+		return "", fmt.Errorf("error creating backup directory: %v", err)
+	}
+	if err := os.Mkdir(filepath.Join(backupDir, t, "data"), 0o755); err != nil {
+		return "", fmt.Errorf("error creating data backup directory: %v", err)
+	}
+	return filepath.Join(backupDir, t, "data"), nil
+}
+
 func GetDirectorySize(dirPath string) (float64, error) {
 	var s int64
 	err := filepath.Walk(dirPath, func(_ string, info os.FileInfo, err error) error {
@@ -47,6 +60,23 @@ func GetLogsDir() (string, error) {
 	return logsDir, nil
 }
 
+func GetBackupDir() (string, error) {
+	home, err := GetSupervysorDir()
+	if err != nil {
+		return "", fmt.Errorf("could not find home directory: %s", err)
+	}
+
+	backupDir := filepath.Join(home, "backups")
+	if _, err = os.Stat(backupDir); os.IsNotExist(err) {
+		err = os.Mkdir(backupDir, 0o755)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return backupDir, nil
+}
+
 func GetSupervysorDir() (string, error) {
 	home, err := os.UserHomeDir()
 	if err != nil {
@@ -128,3 +158,22 @@ func StartMetricsServer(reg *prometheus.Registry) error {
 	}
 	return nil
 }
+
+func ValidatePaths(srcPath, destPath string) error {
+	pathInfo, err := os.Stat(srcPath)
+	if err != nil {
+		return err
+	}
+	if !pathInfo.IsDir() {
+		return err
+	}
+	pathInfo, err = os.Stat(destPath)
+	if err != nil {
+		return err
+	}
+	if !pathInfo.IsDir() {
+		return err
+	}
+
+	return nil
+}

cmd/supervysor/init.go

@@ -21,7 +21,7 @@ var (
 	chainId           string
 	fallbackEndpoints string
 	homePath          string
-	metrics           string
+	metrics           bool
 	poolId            int
 	seeds             string
 	pruningInterval   int
@@ -59,7 +59,7 @@ func init() {
 
 	initCmd.Flags().IntVar(&pruningInterval, "pruning-interval", 24, "block-pruning interval (hours)")
 
-	initCmd.Flags().StringVar(&metrics, "metrics", "true", "exposing Prometheus metrics (true or false)")
+	initCmd.Flags().BoolVar(&metrics, "metrics", true, "exposing Prometheus metrics (true or false)")
 
 	initCmd.Flags().StringVar(&abciEndpoint, "abci-endpoint", "http://127.0.0.1:26657", "ABCI Endpoint to request node information")
 }
@@ -83,7 +83,7 @@ func InitializeSupervysor() error {
 		logger.Error("pruning-interval should be higher than 6 hours")
 	}
 
-	if err := settings.InitializeSettings(binaryPath, homePath, poolId, false, seeds, chainId, fallbackEndpoints, metrics); err != nil {
+	if err := settings.InitializeSettings(binaryPath, homePath, poolId, false, seeds, chainId, fallbackEndpoints); err != nil {
 		logger.Error("could not initialize settings", "err", err)
 		return err
 	}
@@ -107,22 +107,17 @@ func InitializeSupervysor() error {
 		}
 		logger.Info("initializing supverysor...")
 
-		m := true
-		if metrics == "false" {
-			m = false
-		}
-
 		config := types.SupervysorConfig{
 			ABCIEndpoint:        abciEndpoint,
-			ChainId:             chainId,
 			BinaryPath:          binaryPath,
+			ChainId:             chainId,
 			HomePath:            homePath,
+			Interval:            10,
+			Metrics:             metrics,
 			PoolId:              poolId,
 			Seeds:               seeds,
 			FallbackEndpoints:   fallbackEndpoints,
 			PruningInterval:     pruningInterval,
-			Metrics:             m,
-			Interval:            10,
 			HeightDifferenceMax: settings.Settings.MaxDifference,
 			HeightDifferenceMin: settings.Settings.MaxDifference / 2,
 			StateRequests:       false,

cmd/supervysor/main.go

@@ -48,6 +48,7 @@ func main() {
 	supervysor.AddCommand(startCmd)
 	supervysor.AddCommand(versionCmd)
 	supervysor.AddCommand(pruneCmd)
+	supervysor.AddCommand(backupCmd)
 
 	if err = supervysor.Execute(); err != nil {
 		os.Exit(1)

cmd/supervysor/start.go

@@ -95,20 +95,22 @@ var startCmd = &cobra.Command{
 
 			logger.Info("fetched heights successfully", "node", nodeHeight, "pool", poolHeight, "max-height", poolHeight+config.HeightDifferenceMax, "min-height", poolHeight+config.HeightDifferenceMin)
 
-			logger.Info("current pruning count", "pruning-count", fmt.Sprintf("%.2f", pruningCount), "pruning-threshold", config.PruningInterval)
-			if pruningCount > float64(config.PruningInterval) && currentMode == "ghost" && nodeHeight > 0 {
-				pruneHeight := poolHeight
-				if nodeHeight < poolHeight {
-					pruneHeight = nodeHeight
-				}
-				logger.Info("pruning blocks after node shutdown", "until-height", pruneHeight)
+			if config.PruningInterval != 0 {
+				logger.Info("current pruning count", "pruning-count", fmt.Sprintf("%.2f", pruningCount), "pruning-threshold", config.PruningInterval)
+				if pruningCount > float64(config.PruningInterval) && currentMode == "ghost" && nodeHeight > 0 {
+					pruneHeight := poolHeight
+					if nodeHeight < poolHeight {
+						pruneHeight = nodeHeight
+					}
+					logger.Info("pruning blocks after node shutdown", "until-height", pruneHeight)
 
-				err = e.PruneBlocks(config.HomePath, pruneHeight-1, flags)
-				if err != nil {
-					logger.Error("could not prune blocks", "err", err)
-					return err
+					err = e.PruneBlocks(config.HomePath, pruneHeight-1, flags)
+					if err != nil {
+						logger.Error("could not prune blocks", "err", err)
+						return err
+					}
+					pruningCount = 0
 				}
-				pruningCount = 0
 			}
 
 			// Calculate height difference to enable the correct mode.

executor/executor.go

@@ -104,6 +104,7 @@ func (e *Executor) PruneBlocks(homePath string, pruneHeight int, flags []string)
 		e.Logger.Error("could not prune blocks, exiting")
 		return err
 	}
+
 	if e.Process.GhostMode {
 		process, err := node.StartGhostNode(e.Cfg, e.Logger, &e.Process, true, flags)
 		if err != nil {