diff --git a/internal/backup/timestamp.go b/internal/backup/timestamp.go index 5ec265b4..bed15b06 100644 --- a/internal/backup/timestamp.go +++ b/internal/backup/timestamp.go @@ -20,12 +20,13 @@ package backup import ( - "fmt" + "errors" "net/http" "strings" "time" "github.com/majewsky/schwift" + "github.com/sapcc/go-bits/errext" "github.com/sapcc/backup-tools/internal/core" ) @@ -37,14 +38,31 @@ func lastBackupTimestampObj(cfg *core.Configuration) *schwift.Object { // ReadLastBackupTimestamp reads the "last_backup_timestamp" object in Swift to // find when the most recent backup was created. func ReadLastBackupTimestamp(cfg *core.Configuration) (time.Time, error) { - str, err := lastBackupTimestampObj(cfg).Download(nil).AsString() - if err != nil { - if schwift.Is(err, http.StatusNotFound) { - //this branch is esp. relevant for the first ever backup -> we just report a very old last backup to force a backup immediately - return time.Unix(0, 0).UTC(), nil + var str string + + // retry swift download of timestamp file up to 3 times to be more robust + for { + var ( + err error + errs errext.ErrorSet + ) + str, err = lastBackupTimestampObj(cfg).Download(nil).AsString() + if err == nil { + break + } else { + if schwift.Is(err, http.StatusNotFound) { + //this branch is esp. relevant for the first ever backup -> we just report a very old last backup to force a backup immediately + return time.Unix(0, 0).UTC(), nil + } + errs.Addf("could not read last_backup_timestamp from Swift: %w", err) + } + + time.Sleep(1 * time.Second) + if len(errs) == 3 { + return time.Time{}, errors.New(errs.Join(", ")) } - return time.Time{}, fmt.Errorf("could not read last_backup_timestamp from Swift: %w", err) } + t, err := time.ParseInLocation(TimeFormat, str, time.UTC) if err != nil { //recover from malformed timestamp files by forcing a new backup immediately, same as above @@ -57,9 +75,21 @@ func ReadLastBackupTimestamp(cfg *core.Configuration) (time.Time, error) { // to indicate that a backup was completed successfully. func WriteLastBackupTimestamp(cfg *core.Configuration, t time.Time) error { payload := strings.NewReader(t.UTC().Format(TimeFormat)) - err := lastBackupTimestampObj(cfg).Upload(payload, nil, nil) - if err != nil { - return fmt.Errorf("could not write last_backup_timestamp into Swift: %w", err) + + // retry swift upload of timestamp file up to 3 times to be more robust + for { + var errs errext.ErrorSet + err := lastBackupTimestampObj(cfg).Upload(payload, nil, nil) + if err == nil { + break + } else { + errs.Addf("could not write last_backup_timestamp into Swift: %w", err) + } + + time.Sleep(1 * time.Second) + if len(errs) == 3 { + return errors.New(errs.Join(", ")) + } } return nil } diff --git a/vendor/github.com/sapcc/go-bits/errext/errext.go b/vendor/github.com/sapcc/go-bits/errext/errext.go new file mode 100644 index 00000000..fd6f1c58 --- /dev/null +++ b/vendor/github.com/sapcc/go-bits/errext/errext.go @@ -0,0 +1,52 @@ +/******************************************************************************* +* +* Copyright 2023 SAP SE +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You should have received a copy of the License along with this +* program. If not, you may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*******************************************************************************/ + +// Package errext contains convenience functions for handling and propagating errors. +package errext + +import "errors" + +// As is a variant of errors.As() that leverages generics to present a nicer interface. +// +// //this code: +// var perr os.PathError +// if errors.As(err, &perr) { +// handle(perr) +// } +// //can be rewritten as: +// if perr, ok := errext.As[os.PathError](err); ok { +// handle(perr) +// } +// +// This is sometimes more verbose (like in this example), but allows to scope +// the specific error variable to the condition's then-branch, and also looks +// more idiomatic to developers already familiar with type casts. +func As[T error](err error) (T, bool) { + var result T + ok := errors.As(err, &result) + return result, ok +} + +// IsOfType is a variant of errors.As() that only returns whether the match succeeded. +// +// This function is not called Is() to avoid confusion with errors.Is(), which works differently. +func IsOfType[T error](err error) bool { + _, ok := As[T](err) + return ok +} diff --git a/vendor/github.com/sapcc/go-bits/errext/errorset.go b/vendor/github.com/sapcc/go-bits/errext/errorset.go new file mode 100644 index 00000000..409f532d --- /dev/null +++ b/vendor/github.com/sapcc/go-bits/errext/errorset.go @@ -0,0 +1,78 @@ +/******************************************************************************* +* +* Copyright 2023 SAP SE +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You should have received a copy of the License along with this +* program. If not, you may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*******************************************************************************/ + +package errext + +import ( + "fmt" + "os" + "strings" + + "github.com/sapcc/go-bits/logg" +) + +// ErrorSet replaces the "error" return value in functions that can return +// multiple errors. It provides convenience functions for easily adding errors +// to the set. +type ErrorSet []error + +// Add adds the given error to the set if it is non-nil. +func (errs *ErrorSet) Add(err error) { + if err != nil { + *errs = append(*errs, err) + } +} + +// Addf is a shorthand for errs.Add(fmt.Errorf(...)). +func (errs *ErrorSet) Addf(msg string, args ...any) { + *errs = append(*errs, fmt.Errorf(msg, args...)) +} + +// Append adds all errors from the `other` ErrorSet to this one. +func (errs *ErrorSet) Append(other ErrorSet) { + *errs = append(*errs, other...) +} + +// IsEmpty returns true if no errors are in the set. +func (errs ErrorSet) IsEmpty() bool { + return len(errs) == 0 +} + +// Join joins the messages of all errors in this set using the provided separator. +// If the set is empty, an empty string is returned. +func (errs ErrorSet) Join(sep string) string { + msgs := make([]string, len(errs)) + for idx, err := range errs { + msgs[idx] = err.Error() + } + return strings.Join(msgs, sep) +} + +// LogFatalIfError reports all errors in this set on level FATAL, thus dying if +// there are any errors. +func (errs ErrorSet) LogFatalIfError() { + hasErrors := false + for _, err := range errs { + hasErrors = true + logg.Other("FATAL", err.Error()) + } + if hasErrors { + os.Exit(1) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d41fafe0..1dbbc1f3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -70,6 +70,7 @@ github.com/prometheus/procfs/internal/util github.com/sapcc/go-api-declarations/bininfo # github.com/sapcc/go-bits v0.0.0-20240104033923-b834f0c87cf8 ## explicit; go 1.21 +github.com/sapcc/go-bits/errext github.com/sapcc/go-bits/httpapi github.com/sapcc/go-bits/httpext github.com/sapcc/go-bits/logg