Skip to content

Commit

Permalink
Improve download handler interface
Browse files Browse the repository at this point in the history
  • Loading branch information
@koplas
koplas committed Jun 27, 2024
1 parent 0811d11 commit b92da64
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 44 deletions.
56 changes: 49 additions & 7 deletions cmd/csaf_downloader/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ package main

import (
"context"
"encoding/json"
"log/slog"
"os"
"os/signal"
Expand All @@ -19,9 +20,12 @@ import (
"strconv"
"strings"
"sync"
"time"

"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/lib/downloader"
"github.com/csaf-poc/csaf_distribution/v3/util"
)

// failedForwardDir is the name of the special sub folder
Expand Down Expand Up @@ -79,6 +83,35 @@ func mkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}

func extractInitialReleaseDate(doc any) time.Time {
var initialReleaseDate time.Time
dateExtract := util.TimeMatcher(&initialReleaseDate, time.RFC3339)

eval := util.NewPathEval()

if err := eval.Extract(
`$.document.tracking.initial_release_date`, dateExtract, false, doc,
); err != nil {
slog.Warn("Cannot extract initial_release_date from advisory")
initialReleaseDate = time.Now()
}
initialReleaseDate = initialReleaseDate.UTC()
return initialReleaseDate
}

func extractTLP(doc any) csaf.TLPLabel {
eval := util.NewPathEval()
labelString, err := eval.Eval(`$.document.distribution.tlp.label`, doc)
if err != nil {
return csaf.TLPLabelUnlabeled
}
label, ok := labelString.(string)
if !ok {
return csaf.TLPLabelUnlabeled
}
return csaf.TLPLabel(label)
}

func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error {
return func(d downloader.DownloadedDocument) error {
if cfg.NoStore {
Expand All @@ -98,13 +131,22 @@ func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error {
newDir = cfg.Directory
}

lower := strings.ToLower(string(d.Label))
var doc any
if err := json.Unmarshal(d.Data, &doc); err != nil {
slog.Error("Could not parse json document", "err", err)
return nil
}

initialReleaseDate := extractInitialReleaseDate(doc)
label := extractTLP(doc)

lower := strings.ToLower(string(label))

// Do we have a configured destination folder?
if cfg.Folder != "" {
newDir = path.Join(newDir, cfg.Folder)
} else {
newDir = path.Join(newDir, lower, strconv.Itoa(d.InitialReleaseDate.Year()))
newDir = path.Join(newDir, lower, strconv.Itoa(initialReleaseDate.Year()))
}

if newDir != lastDir {
Expand All @@ -121,9 +163,9 @@ func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error {
p string
d []byte
}{
{filePath, d.Data.Bytes()},
{filePath + ".sha256", d.S256Data},
{filePath + ".sha512", d.S512Data},
{filePath, d.Data},
{filePath + ".sha256", d.SHA256},
{filePath + ".sha512", d.SHA512},
{filePath + ".asc", d.SignData},
} {
if x.d != nil {
Expand Down Expand Up @@ -157,8 +199,8 @@ func storeFailedAdvisory(cfg *config) func(filename, doc, sha256, sha512 string)
{filename + ".sha512", sha512},
} {
if len(x.d) != 0 {
path := filepath.Join(dir, x.p)
if err := os.WriteFile(path, []byte(x.d), 0644); err != nil {
p := filepath.Join(dir, x.p)
if err := os.WriteFile(p, []byte(x.d), 0644); err != nil {
return err
}
}
Expand Down
55 changes: 18 additions & 37 deletions lib/downloader/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
type Downloader struct {
cfg *Config
keys *crypto.KeyRing
eval *util.PathEval
validator csaf.RemoteValidator
Forwarder *Forwarder
mkdirMu sync.Mutex
Expand All @@ -47,14 +48,12 @@ type Downloader struct {

// DownloadedDocument contains the document data with additional metadata.
type DownloadedDocument struct {
Data bytes.Buffer
S256Data []byte
S512Data []byte
SignData []byte
InitialReleaseDate time.Time
Filename string
ValStatus ValidationStatus
Label csaf.TLPLabel
Data []byte
SHA256 []byte
SHA512 []byte
SignData []byte
Filename string
ValStatus ValidationStatus
}

// failedValidationDir is the name of the sub folder
Expand All @@ -64,7 +63,6 @@ const failedValidationDir = "failed_validation"

// NewDownloader constructs a new downloader given the configuration.
func NewDownloader(cfg *Config) (*Downloader, error) {

var validator csaf.RemoteValidator

if cfg.RemoteValidator != "" {
Expand Down Expand Up @@ -117,7 +115,6 @@ func logRedirect(logger *slog.Logger) func(req *http.Request, via []*http.Reques
}

func (d *Downloader) httpClient() util.Client {

hClient := http.Client{}

if d.cfg.verbose() {
Expand Down Expand Up @@ -255,16 +252,14 @@ func (d *Downloader) download(ctx context.Context, domain string) error {
}

return afp.Process(func(label csaf.TLPLabel, files []csaf.AdvisoryFile) error {
return d.downloadFiles(ctx, label, files)
return d.downloadFiles(ctx, files)
})
}

func (d *Downloader) downloadFiles(
ctx context.Context,
label csaf.TLPLabel,
files []csaf.AdvisoryFile,
) error {

var (
advisoryCh = make(chan csaf.AdvisoryFile)
errorCh = make(chan error)
Expand All @@ -288,7 +283,7 @@ func (d *Downloader) downloadFiles(

for i := 0; i < n; i++ {
wg.Add(1)
go d.downloadWorker(ctx, &wg, label, advisoryCh, errorCh)
go d.downloadWorker(ctx, &wg, advisoryCh, errorCh)
}

allFiles:
Expand Down Expand Up @@ -367,7 +362,6 @@ func (d *Downloader) loadOpenPGPKeys(
defer res.Body.Close()
return crypto.NewKeyFromArmoredReader(res.Body)
}()

if err != nil {
d.cfg.Logger.Warn(
"Reading public OpenPGP key failed",
Expand Down Expand Up @@ -422,18 +416,15 @@ func (d *Downloader) logValidationIssues(url string, errors []string, err error)
func (d *Downloader) downloadWorker(
ctx context.Context,
wg *sync.WaitGroup,
label csaf.TLPLabel,
files <-chan csaf.AdvisoryFile,
errorCh chan<- error,
) {
defer wg.Done()

var (
client = d.httpClient()
data bytes.Buffer
initialReleaseDate time.Time
dateExtract = util.TimeMatcher(&initialReleaseDate, time.RFC3339)
stats = stats{}
client = d.httpClient()
data bytes.Buffer
stats = stats{}
expr = util.NewPathEval()
)

Expand Down Expand Up @@ -656,24 +647,14 @@ nextAdvisory:
string(s256Data),
string(s512Data))
}
if err := expr.Extract(
`$.document.tracking.initial_release_date`, dateExtract, false, doc,
); err != nil {
slog.Warn("Cannot extract initial_release_date from advisory",
"url", file.URL())
initialReleaseDate = time.Now()
}
initialReleaseDate = initialReleaseDate.UTC()

download := DownloadedDocument{
Data: data,
S256Data: s256Data,
S512Data: s512Data,
SignData: signData,
InitialReleaseDate: initialReleaseDate,
Filename: filename,
ValStatus: valStatus,
Label: label,
Data: data.Bytes(),
SHA256: s256Data,
SHA512: s512Data,
SignData: signData,
Filename: filename,
ValStatus: valStatus,
}

err = d.cfg.DownloadHandler(download)
Expand Down

0 comments on commit b92da64

Please sign in to comment.