diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 3bfea979..e43686b8 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -11,7 +11,11 @@ package main import ( "crypto/tls" "fmt" + "io" + "log/slog" "net/http" + "os" + "path/filepath" "github.com/csaf-poc/csaf_distribution/v2/internal/certs" "github.com/csaf-poc/csaf_distribution/v2/internal/filter" @@ -24,6 +28,8 @@ const ( defaultPreset = "mandatory" defaultForwardQueue = 5 defaultValidationMode = validationStrict + defaultLogFile = "downloader.log" + defaultLogLevel = logLevelInfo ) type validationMode string @@ -33,8 +39,17 @@ const ( validationUnsafe = validationMode("unsafe") ) +type logLevel string + +const ( + logLevelDebug = logLevel("debug") + logLevelInfo = logLevel("info") + logLevelWarn = logLevel("warn") + logLevelError = logLevel("error") +) + type config struct { - Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"` + Directory string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"` Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"` IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignoresigcheck"` ClientCert *string `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE" toml:"client_cert"` @@ -62,6 +77,10 @@ type config struct { ForwardQueue int `long:"forwardqueue" description:"Maximal queue LENGTH before forwarder" value-name:"LENGTH" toml:"forward_queue"` ForwardInsecure bool `long:"forwardinsecure" description:"Do not check TLS certificates from forward endpoint" toml:"forward_insecure"` + LogFile string `long:"logfile" description:"FILE to log download to" value-name:"FILE" toml:"log_file"` + //lint:ignore SA5008 We are using choice or than once: debug, info, warn, error + LogLevel logLevel `long:"loglevel" description:"LEVEL of logging details" value-name:"LEVEL" choice:"debug" choice:"info" choice:"warn" choice:"error" toml:"log_level"` + Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"` clientCerts []tls.Certificate @@ -87,6 +106,8 @@ func parseArgsConfig() ([]string, *config, error) { cfg.RemoteValidatorPresets = []string{defaultPreset} cfg.ValidationMode = defaultValidationMode cfg.ForwardQueue = defaultForwardQueue + cfg.LogFile = defaultLogFile + cfg.LogLevel = defaultLogLevel }, // Re-establish default values if not set. EnsureDefaults: func(cfg *config) { @@ -117,11 +138,94 @@ func (vm *validationMode) UnmarshalText(text []byte) error { return nil } +// UnmarshalText implements [encoding/text.TextUnmarshaler]. +func (ll *logLevel) UnmarshalText(text []byte) error { + switch l := logLevel(text); l { + case logLevelDebug, logLevelInfo, logLevelWarn, logLevelError: + *ll = l + default: + return fmt.Errorf(`invalid value %q (expected "debug", "info", "warn", "error")`, l) + } + return nil +} + // ignoreFile returns true if the given URL should not be downloaded. func (cfg *config) ignoreURL(u string) bool { return cfg.ignorePattern.Matches(u) } +// slogLevel converts logLevel to [slog.Level]. +func (ll logLevel) slogLevel() slog.Level { + switch ll { + case logLevelDebug: + return slog.LevelDebug + case logLevelInfo: + return slog.LevelInfo + case logLevelWarn: + return slog.LevelWarn + case logLevelError: + return slog.LevelError + default: + return slog.LevelInfo + } +} + +// prepareDirectory ensures that the working directory +// exists and is setup properly. +func (cfg *config) prepareDirectory() error { + // If no special given use current working directory. + if cfg.Directory == "" { + dir, err := os.Getwd() + if err != nil { + return err + } + cfg.Directory = dir + return nil + } + // Use given directory + if _, err := os.Stat(cfg.Directory); err != nil { + // If it does not exist create it. + if os.IsNotExist(err) { + if err = os.MkdirAll(cfg.Directory, 0755); err != nil { + return err + } + } else { + return err + } + } + return nil +} + +// prepareLogging sets up the structured logging. +func (cfg *config) prepareLogging() error { + var w io.Writer + if cfg.LogFile == "" { + w = os.Stderr + } else { + var fname string + // We put the log inside the download folder + // if it is not absolute. + if filepath.IsAbs(cfg.LogFile) { + fname = cfg.LogFile + } else { + fname = filepath.Join(cfg.Directory, cfg.LogFile) + } + f, err := os.OpenFile(fname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + if err != nil { + return err + } + w = f + } + ho := slog.HandlerOptions{ + //AddSource: true, + Level: cfg.LogLevel.slogLevel(), + } + handler := slog.NewJSONHandler(w, &ho) + logger := slog.New(handler) + slog.SetDefault(logger) + return nil +} + // compileIgnorePatterns compiles the configure patterns to be ignored. func (cfg *config) compileIgnorePatterns() error { pm, err := filter.NewPatternMatcher(cfg.IgnorePattern) @@ -145,8 +249,15 @@ func (cfg *config) prepareCertificates() error { // prepare prepares internal state of a loaded configuration. func (cfg *config) prepare() error { - if err := cfg.prepareCertificates(); err != nil { - return err + for _, prepare := range []func(*config) error{ + (*config).prepareDirectory, + (*config).prepareLogging, + (*config).prepareCertificates, + (*config).compileIgnorePatterns, + } { + if err := prepare(cfg); err != nil { + return err + } } - return cfg.compileIgnorePatterns() + return nil } diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 68bf837d..ace7653f 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -3,8 +3,8 @@ // // SPDX-License-Identifier: MIT // -// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) -// Software-Engineering: 2022 Intevation GmbH +// SPDX-FileCopyrightText: 2022, 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2022, 2023 Intevation GmbH package main @@ -19,7 +19,7 @@ import ( "fmt" "hash" "io" - "log" + "log/slog" "net/http" "net/url" "os" @@ -31,21 +31,28 @@ import ( "time" "github.com/ProtonMail/gopenpgp/v2/crypto" + "golang.org/x/time/rate" + "github.com/csaf-poc/csaf_distribution/v2/csaf" "github.com/csaf-poc/csaf_distribution/v2/util" - "golang.org/x/time/rate" ) type downloader struct { cfg *config - directory string keys *crypto.KeyRing eval *util.PathEval validator csaf.RemoteValidator forwarder *forwarder mkdirMu sync.Mutex + statsMu sync.Mutex + stats stats } +// failedValidationDir is the name of the sub folder +// where advisories are stored that fail validation in +// unsafe mode. +const failedValidationDir = "failed_validation" + func newDownloader(cfg *config) (*downloader, error) { var validator csaf.RemoteValidator @@ -78,6 +85,13 @@ func (d *downloader) close() { } } +// addStats add stats to total stats +func (d *downloader) addStats(o *stats) { + d.statsMu.Lock() + defer d.statsMu.Unlock() + d.stats.add(o) +} + func (d *downloader) httpClient() util.Client { hClient := http.Client{} @@ -130,8 +144,9 @@ func (d *downloader) download(ctx context.Context, domain string) error { if d.cfg.Verbose { for i := range lpmd.Messages { - log.Printf("Loading provider-metadata.json for %q: %s\n", - domain, lpmd.Messages[i].Message) + slog.Info("Loading provider-metadata.json", + "domain", domain, + "message", lpmd.Messages[i].Message) } } @@ -247,7 +262,9 @@ func (d *downloader) loadOpenPGPKeys( } up, err := url.Parse(*key.URL) if err != nil { - log.Printf("Invalid URL '%s': %v", *key.URL, err) + slog.Warn("Invalid URL", + "url", *key.URL, + "error", err) continue } @@ -255,12 +272,18 @@ func (d *downloader) loadOpenPGPKeys( res, err := client.Get(u) if err != nil { - log.Printf("Fetching public OpenPGP key %s failed: %v.", u, err) + slog.Warn( + "Fetching public OpenPGP key failed", + "url", u, + "error", err) continue } if res.StatusCode != http.StatusOK { - log.Printf("Fetching public OpenPGP key %s status code: %d (%s)", - u, res.StatusCode, res.Status) + slog.Warn( + "Fetching public OpenPGP key failed", + "url", u, + "status_code", res.StatusCode, + "status", res.Status) continue } @@ -270,18 +293,25 @@ func (d *downloader) loadOpenPGPKeys( }() if err != nil { - log.Printf("Reading public OpenPGP key %s failed: %v", u, err) + slog.Warn( + "Reading public OpenPGP key failed", + "url", u, + "error", err) continue } if !strings.EqualFold(ckey.GetFingerprint(), string(key.Fingerprint)) { - log.Printf( - "Fingerprint of public OpenPGP key %s does not match remotely loaded.", u) + slog.Warn( + "Fingerprint of public OpenPGP key does not match remotely loaded", + "url", u) continue } if d.keys == nil { if keyring, err := crypto.NewKeyRing(ckey); err != nil { - log.Printf("Creating store for public OpenPGP key %s failed: %v.", u, err) + slog.Warn( + "Creating store for public OpenPGP key failed", + "url", u, + "error", err) } else { d.keys = keyring } @@ -295,16 +325,20 @@ func (d *downloader) loadOpenPGPKeys( // logValidationIssues logs the issues reported by the advisory schema validation. func (d *downloader) logValidationIssues(url string, errors []string, err error) { if err != nil { - log.Printf("Failed to validate %s: %v", url, err) + slog.Error("Failed to validate", + "url", url, + "error", err) return } if len(errors) > 0 { if d.cfg.Verbose { - log.Printf("CSAF file %s has validation errors: %s\n", - url, strings.Join(errors, ", ")) + slog.Error("CSAF file has validation errors", + "url", url, + "error", strings.Join(errors, ", ")) } else { - log.Printf("CSAF file %s has %d validation errors.\n", - url, len(errors)) + slog.Error("CSAF file has validation errors", + "url", url, + "count", len(errors)) } } } @@ -325,8 +359,12 @@ func (d *downloader) downloadWorker( initialReleaseDate time.Time dateExtract = util.TimeMatcher(&initialReleaseDate, time.RFC3339) lower = strings.ToLower(string(label)) + stats = stats{} ) + // Add collected stats back to total. + defer d.addStats(&stats) + nextAdvisory: for { var file csaf.AdvisoryFile @@ -342,41 +380,52 @@ nextAdvisory: u, err := url.Parse(file.URL()) if err != nil { - log.Printf("Ignoring invalid URL: %s: %v\n", file.URL(), err) - continue - } - - // Ignore not conforming filenames. - filename := filepath.Base(u.Path) - if !util.ConformingFileName(filename) { - log.Printf("Not conforming filename %q. Ignoring.\n", filename) + stats.downloadFailed++ + slog.Warn("Ignoring invalid URL", + "url", file.URL(), + "error", err) continue } if d.cfg.ignoreURL(file.URL()) { if d.cfg.Verbose { - log.Printf("Ignoring %q.\n", file.URL()) + slog.Warn("Ignoring URL", "url", file.URL()) } continue } + // Ignore not conforming filenames. + filename := filepath.Base(u.Path) + if !util.ConformingFileName(filename) { + stats.filenameFailed++ + slog.Warn("Ignoring none conforming filename", + "filename", filename) + continue + } + resp, err := client.Get(file.URL()) if err != nil { - log.Printf("WARN: cannot get '%s': %v\n", file.URL(), err) + stats.downloadFailed++ + slog.Warn("Cannot GET", + "url", file.URL(), + "error", err) continue } if resp.StatusCode != http.StatusOK { - log.Printf("WARN: cannot load %s: %s (%d)\n", - file.URL(), resp.Status, resp.StatusCode) + stats.downloadFailed++ + slog.Warn("Cannot load", + "url", file.URL(), + "status", resp.Status, + "status_code", resp.StatusCode) continue } // Warn if we do not get JSON. if ct := resp.Header.Get("Content-Type"); ct != "application/json" { - log.Printf( - "WARN: The content type of %s should be 'application/json' but is '%s'\n", - file.URL(), ct) + slog.Warn("Content type is not 'application/json'", + "url", file.URL(), + "content_type", ct) } var ( @@ -390,7 +439,9 @@ nextAdvisory: // Only hash when we have a remote counter part we can compare it with. if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil { if d.cfg.Verbose { - log.Printf("WARN: cannot fetch %s: %v\n", file.SHA256URL(), err) + slog.Warn("Cannot fetch SHA256", + "url", file.SHA256URL(), + "error", err) } } else { s256 = sha256.New() @@ -399,7 +450,9 @@ nextAdvisory: if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil { if d.cfg.Verbose { - log.Printf("WARN: cannot fetch %s: %v\n", file.SHA512URL(), err) + slog.Warn("Cannot fetch SHA512", + "url", file.SHA512URL(), + "error", err) } } else { s512 = sha512.New() @@ -420,13 +473,17 @@ nextAdvisory: tee := io.TeeReader(resp.Body, hasher) return json.NewDecoder(tee).Decode(&doc) }(); err != nil { - log.Printf("Downloading %s failed: %v", file.URL(), err) + stats.downloadFailed++ + slog.Warn("Downloading failed", + "url", file.URL(), + "error", err) continue } // Compare the checksums. s256Check := func() error { if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) { + stats.sha256Failed++ return fmt.Errorf("SHA256 checksum of %s does not match", file.URL()) } return nil @@ -434,12 +491,13 @@ nextAdvisory: s512Check := func() error { if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) { + stats.sha512Failed++ return fmt.Errorf("SHA512 checksum of %s does not match", file.URL()) } return nil } - // Validate OpenPGG signature. + // Validate OpenPGP signature. keysCheck := func() error { // Only check signature if we have loaded keys. if d.keys == nil { @@ -449,13 +507,15 @@ nextAdvisory: sign, signData, err = loadSignature(client, file.SignURL()) if err != nil { if d.cfg.Verbose { - log.Printf("downloading signature '%s' failed: %v\n", - file.SignURL(), err) + slog.Warn("Downloading signature failed", + "url", file.SignURL(), + "error", err) } } if sign != nil { if err := d.checkSignature(data.Bytes(), sign); err != nil { if !d.cfg.IgnoreSignatureCheck { + stats.signatureFailed++ return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err) } } @@ -466,6 +526,7 @@ nextAdvisory: // Validate against CSAF schema. schemaCheck := func() error { if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 { + stats.schemaFailed++ d.logValidationIssues(file.URL(), errors, err) return fmt.Errorf("schema validation for %q failed", file.URL()) } @@ -475,6 +536,7 @@ nextAdvisory: // Validate if filename is conforming. filenameCheck := func() error { if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil { + stats.filenameFailed++ return fmt.Errorf("filename not conforming %s: %s", file.URL(), err) } return nil @@ -493,6 +555,7 @@ nextAdvisory: return nil } if !rvr.Valid { + stats.remoteFailed++ return fmt.Errorf("remote validation of %q failed", file.URL()) } return nil @@ -509,8 +572,7 @@ nextAdvisory: remoteValidatorCheck, } { if err := check(); err != nil { - // TODO: Improve logging. - log.Printf("check failed: %v\n", err) + slog.Error("Validation check failed", "error", err) valStatus.update(invalidValidationStatus) if d.cfg.ValidationMode == validationStrict { continue nextAdvisory @@ -530,17 +592,26 @@ nextAdvisory: if d.cfg.NoStore { // Do not write locally. + if valStatus == validValidationStatus { + stats.succeeded++ + } continue } if err := d.eval.Extract(`$.document.tracking.initial_release_date`, dateExtract, false, doc); err != nil { - log.Printf("Cannot extract initial_release_date from advisory '%s'\n", file.URL()) + slog.Warn("Cannot extract initial_release_date from advisory", + "url", file.URL()) initialReleaseDate = time.Now() } initialReleaseDate = initialReleaseDate.UTC() - // Write advisory to file - newDir := path.Join(d.directory, lower) + // Advisories that failed validation are store in a special folder. + var newDir string + if valStatus != validValidationStatus { + newDir = path.Join(d.cfg.Directory, failedValidationDir, lower) + } else { + newDir = path.Join(d.cfg.Directory, lower) + } // Do we have a configured destination folder? if d.cfg.Folder != "" { @@ -557,6 +628,7 @@ nextAdvisory: lastDir = newDir } + // Write advisory to file path := filepath.Join(lastDir, filename) // Write data to disk. @@ -577,7 +649,8 @@ nextAdvisory: } } - log.Printf("Written advisory '%s'.\n", path) + stats.succeeded++ + slog.Info("Written advisory", "path", path) } } @@ -633,40 +706,9 @@ func loadHash(client util.Client, p string) ([]byte, []byte, error) { return hash, data.Bytes(), nil } -// prepareDirectory ensures that the working directory -// exists and is setup properly. -func (d *downloader) prepareDirectory() error { - // If no special given use current working directory. - if d.cfg.Directory == nil { - dir, err := os.Getwd() - if err != nil { - return err - } - d.directory = dir - return nil - } - // Use given directory - if _, err := os.Stat(*d.cfg.Directory); err != nil { - // If it does not exist create it. - if os.IsNotExist(err) { - if err = os.MkdirAll(*d.cfg.Directory, 0755); err != nil { - return err - } - } else { - return err - } - } - d.directory = *d.cfg.Directory - return nil -} - // run performs the downloads for all the given domains. func (d *downloader) run(ctx context.Context, domains []string) error { - - if err := d.prepareDirectory(); err != nil { - return err - } - + defer d.stats.log() for _, domain := range domains { if err := d.download(ctx, domain); err != nil { return err diff --git a/cmd/csaf_downloader/forwarder.go b/cmd/csaf_downloader/forwarder.go index c84a1313..6efdfbe2 100644 --- a/cmd/csaf_downloader/forwarder.go +++ b/cmd/csaf_downloader/forwarder.go @@ -12,9 +12,10 @@ import ( "bytes" "crypto/tls" "io" - "log" + "log/slog" "mime/multipart" "net/http" + "os" "path/filepath" "strings" @@ -22,6 +23,10 @@ import ( "github.com/csaf-poc/csaf_distribution/v2/util" ) +// failedForwardDir is the name of the special sub folder +// where advisories get stored which fail forwarding. +const failedForwardDir = "failed_forward" + // validationStatus represents the validation status // known to the HTTP endpoint. type validationStatus string @@ -45,6 +50,9 @@ type forwarder struct { cfg *config cmds chan func(*forwarder) client util.Client + + failed int + succeeded int } // newForwarder creates a new forwarder. @@ -58,7 +66,7 @@ func newForwarder(cfg *config) *forwarder { // run runs the forwarder. Meant to be used in a Go routine. func (f *forwarder) run() { - defer log.Println("debug: forwarder done") + defer slog.Debug("forwarder done") for cmd := range f.cmds { cmd(f) @@ -70,6 +78,15 @@ func (f *forwarder) close() { close(f.cmds) } +// log logs the current statistics. +func (f *forwarder) log() { + f.cmds <- func(f *forwarder) { + slog.Info("Forward statistics", + "succeeded", f.succeeded, + "failed", f.failed) + } +} + // httpClient returns a cached HTTP client used for uploading // the advisories to the configured HTTP endpoint. func (f *forwarder) httpClient() util.Client { @@ -113,76 +130,124 @@ func replaceExt(fname, nExt string) string { return fname[:len(fname)-len(ext)] + nExt } -// forward sends a given document with filename, status and -// checksums to the forwarder. This is async to the degree -// till the configured queue size is filled. -func (f *forwarder) forward( +// buildRequest creates an HTTP request suited ti forward the given advisory. +func (f *forwarder) buildRequest( filename, doc string, status validationStatus, sha256, sha512 string, -) { - buildRequest := func() (*http.Request, error) { - body := new(bytes.Buffer) - writer := multipart.NewWriter(body) - - var err error - part := func(name, fname, mimeType, content string) { - if err != nil { - return - } - if fname == "" { - err = writer.WriteField(name, content) - return - } - var w io.Writer - if w, err = misc.CreateFormFile(writer, name, fname, mimeType); err == nil { - _, err = w.Write([]byte(content)) - } - } +) (*http.Request, error) { + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) - base := filepath.Base(filename) - part("advisory", base, "application/json", doc) - part("validation_status", "", "text/plain", string(status)) - if sha256 != "" { - part("hash-256", replaceExt(base, ".sha256"), "text/plain", sha256) + var err error + part := func(name, fname, mimeType, content string) { + if err != nil { + return } - if sha512 != "" { - part("hash-512", replaceExt(base, ".sha512"), "text/plain", sha512) + if fname == "" { + err = writer.WriteField(name, content) + return } - - if err != nil { - return nil, err + var w io.Writer + if w, err = misc.CreateFormFile(writer, name, fname, mimeType); err == nil { + _, err = w.Write([]byte(content)) } + } - if err := writer.Close(); err != nil { - return nil, err - } + base := filepath.Base(filename) + part("advisory", base, "application/json", doc) + part("validation_status", "", "text/plain", string(status)) + if sha256 != "" { + part("hash-256", replaceExt(base, ".sha256"), "text/plain", sha256) + } + if sha512 != "" { + part("hash-512", replaceExt(base, ".sha512"), "text/plain", sha512) + } - req, err := http.NewRequest(http.MethodPost, f.cfg.ForwardURL, body) - if err != nil { - return nil, err + if err != nil { + return nil, err + } + + if err := writer.Close(); err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, f.cfg.ForwardURL, body) + if err != nil { + return nil, err + } + contentType := writer.FormDataContentType() + req.Header.Set("Content-Type", contentType) + return req, nil +} + +// storeFailedAdvisory stores an advisory in a special folder +// in case the forwarding failed. +func (f *forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) error { + dir := filepath.Join(f.cfg.Directory, failedForwardDir) + // Create special folder if it does not exist. + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } else { + return err + } + } + // Store parts which are not empty. + for _, x := range []struct { + p string + d string + }{ + {filename, doc}, + {filename + ".sha256", sha256}, + {filename + ".sha512", sha512}, + } { + if len(x.d) != 0 { + path := filepath.Join(dir, x.p) + if err := os.WriteFile(path, []byte(x.d), 0644); err != nil { + return err + } } - contentType := writer.FormDataContentType() - req.Header.Set("Content-Type", contentType) - return req, nil } + return nil +} + +// storeFailed is a logging wrapper around storeFailedAdvisory. +func (f *forwarder) storeFailed(filename, doc, sha256, sha512 string) { + f.failed++ + if err := f.storeFailedAdvisory(filename, doc, sha256, sha512); err != nil { + slog.Error("Storing advisory failed forwarding failed", + "error", err) + } +} +// forward sends a given document with filename, status and +// checksums to the forwarder. This is async to the degree +// till the configured queue size is filled. +func (f *forwarder) forward( + filename, doc string, + status validationStatus, + sha256, sha512 string, +) { // Run this in the main loop of the forwarder. f.cmds <- func(f *forwarder) { - req, err := buildRequest() + req, err := f.buildRequest(filename, doc, status, sha256, sha512) if err != nil { - // TODO: improve logging - log.Printf("error: %v\n", err) + slog.Error("building forward Request failed", + "error", err) + f.storeFailed(filename, doc, sha256, sha512) return } res, err := f.httpClient().Do(req) if err != nil { - // TODO: improve logging - log.Printf("error: %v\n", err) + slog.Error("sending forward request failed", + "error", err) + f.storeFailed(filename, doc, sha256, sha512) return } if res.StatusCode != http.StatusCreated { - // TODO: improve logging defer res.Body.Close() var msg strings.Builder io.Copy(&msg, io.LimitReader(res.Body, 512)) @@ -190,10 +255,16 @@ func (f *forwarder) forward( if msg.Len() >= 512 { dots = "..." } - log.Printf("error: %s: %q (%d)\n", - filename, msg.String()+dots, res.StatusCode) + slog.Error("forwarding failed", + "filename", filename, + "body", msg.String()+dots, + "status_code", res.StatusCode) + f.storeFailed(filename, doc, sha256, sha512) } else { - log.Printf("info: forwarding %q succeeded\n", filename) + f.succeeded++ + slog.Debug( + "forwarding succeeded", + "filename", filename) } } } diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index fa8b2322..5840791c 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -11,7 +11,7 @@ package main import ( "context" - "log" + "log/slog" "os" "os/signal" @@ -33,7 +33,10 @@ func run(cfg *config, domains []string) error { if cfg.ForwardURL != "" { f := newForwarder(cfg) go f.run() - defer f.close() + defer func() { + f.log() + f.close() + }() d.forwarder = f } @@ -47,7 +50,7 @@ func main() { options.ErrorCheck(cfg.prepare()) if len(domains) == 0 { - log.Println("No domains given.") + slog.Info("No domains given.") return } diff --git a/cmd/csaf_downloader/stats.go b/cmd/csaf_downloader/stats.go new file mode 100644 index 00000000..6bafbf7b --- /dev/null +++ b/cmd/csaf_downloader/stats.go @@ -0,0 +1,59 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +package main + +import "log/slog" + +// stats contains counters of the downloads. +type stats struct { + downloadFailed int + filenameFailed int + schemaFailed int + remoteFailed int + sha256Failed int + sha512Failed int + signatureFailed int + succeeded int +} + +// add adds other stats to this. +func (st *stats) add(o *stats) { + st.downloadFailed += o.downloadFailed + st.filenameFailed += o.filenameFailed + st.schemaFailed += o.schemaFailed + st.remoteFailed += o.remoteFailed + st.sha256Failed += o.sha256Failed + st.sha512Failed += o.sha512Failed + st.signatureFailed += o.signatureFailed + st.succeeded += o.succeeded +} + +func (st *stats) totalFailed() int { + return st.downloadFailed + + st.filenameFailed + + st.schemaFailed + + st.remoteFailed + + st.sha256Failed + + st.sha512Failed + + st.signatureFailed +} + +// log logs the collected stats. +func (st *stats) log() { + slog.Info("Download statistics", + "succeeded", st.succeeded, + "total_failed", st.totalFailed(), + "filename_failed", st.filenameFailed, + "download_failed", st.downloadFailed, + "schema_failed", st.schemaFailed, + "remote_failed", st.remoteFailed, + "sha256_failed", st.sha256Failed, + "sha512_failed", st.sha512Failed, + "signature_failed", st.signatureFailed) +}