From 0ab851a87428ddce7a55a335bd0d58e8dc541e73 Mon Sep 17 00:00:00 2001 From: koplas <54645365+koplas@users.noreply.github.com> Date: Thu, 25 Jul 2024 13:26:29 +0200 Subject: [PATCH 1/4] Use a default user agent --- cmd/csaf_aggregator/config.go | 5 +++++ cmd/csaf_checker/processor.go | 8 +++----- cmd/csaf_downloader/downloader.go | 8 +++----- cmd/csaf_downloader/forwarder.go | 8 +++----- util/client.go | 5 +++++ 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go index 98085424..b73286c3 100644 --- a/cmd/csaf_aggregator/config.go +++ b/cmd/csaf_aggregator/config.go @@ -301,6 +301,11 @@ func (c *config) httpClient(p *provider) util.Client { Client: client, Header: c.ExtraHeader, } + default: + client = &util.HeaderClient{ + Client: client, + Header: http.Header{}, + } } if c.Verbose { diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 451a315c..49e815cd 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -435,11 +435,9 @@ func (p *processor) fullClient() util.Client { client := util.Client(&hClient) // Add extra headers. - if len(p.cfg.ExtraHeader) > 0 { - client = &util.HeaderClient{ - Client: client, - Header: p.cfg.ExtraHeader, - } + client = &util.HeaderClient{ + Client: client, + Header: p.cfg.ExtraHeader, } // Add optional URL logging. diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index badf0605..9cef294c 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -126,11 +126,9 @@ func (d *downloader) httpClient() util.Client { client := util.Client(&hClient) // Add extra headers. - if len(d.cfg.ExtraHeader) > 0 { - client = &util.HeaderClient{ - Client: client, - Header: d.cfg.ExtraHeader, - } + client = &util.HeaderClient{ + Client: client, + Header: d.cfg.ExtraHeader, } // Add optional URL logging. diff --git a/cmd/csaf_downloader/forwarder.go b/cmd/csaf_downloader/forwarder.go index 3b1435a3..c3681eba 100644 --- a/cmd/csaf_downloader/forwarder.go +++ b/cmd/csaf_downloader/forwarder.go @@ -111,11 +111,9 @@ func (f *forwarder) httpClient() util.Client { client := util.Client(&hClient) // Add extra headers. - if len(f.cfg.ForwardHeader) > 0 { - client = &util.HeaderClient{ - Client: client, - Header: f.cfg.ForwardHeader, - } + client = &util.HeaderClient{ + Client: client, + Header: f.cfg.ForwardHeader, } // Add optional URL logging. diff --git a/util/client.go b/util/client.go index 5a11c7b3..441aaaa1 100644 --- a/util/client.go +++ b/util/client.go @@ -61,6 +61,11 @@ func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) { req.Header.Add(key, v) } } + + // Use default user agent if none is set + if userAgent := hc.Header.Get("User-Agent"); userAgent == "" { + req.Header.Add("User-Agent", "csaf-distribution/"+SemVersion) + } return hc.Client.Do(req) } From 3a67fb52100dac0ca64719899afb431fbb8bd590 Mon Sep 17 00:00:00 2001 From: koplas <54645365+koplas@users.noreply.github.com> Date: Wed, 31 Jul 2024 11:00:40 +0200 Subject: [PATCH 2/4] Add user-agent documentation --- docs/csaf_checker.md | 13 +++++++++---- docs/csaf_downloader.md | 13 +++++++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index 58f77cab..a5bc0bf1 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -30,9 +30,12 @@ Help Options: Will check all given _domains_, by trying each as a CSAF provider. +If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf-distribution/version` is sent. + If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and checking proceeds from there. If no config file is explictly given the follwing places are searched for a config file: + ``` ~/.config/csaf/checker.toml ~/.csaf_checker.toml @@ -41,6 +44,7 @@ csaf_checker.toml with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems. Supported options in config files: + ``` output = "" format = "json" @@ -58,9 +62,10 @@ validator_preset = ["mandatory"] ``` Usage example: -` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html` +`./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html` Each performed check has a return type of either 0,1 or 2: + ``` type 0: success type 1: warning @@ -70,16 +75,16 @@ type 2: error The checker result is a success if no checks resulted in type 2, and a failure otherwise. The option `timerange` allows to only check advisories from a given time -interval. It can only be given once. See the +interval. It can only be given once. See the [downloader documentation](csaf_downloader.md#timerange-option) for details. - You can ignore certain advisories while checking by specifying a list of regular expressions[^1] to match their URLs by using the `ignorepattern` option. E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain the sub strings **white** or **red**. In the config file this has to be noted as: + ``` ignorepattern = [".*white.*", ".*red.*"] ``` @@ -88,7 +93,7 @@ ignorepattern = [".*white.*", ".*red.*"] The `role` given in the `provider-metadata.json` is not yet considered to change the overall result, -see https://github.com/csaf-poc/csaf_distribution/issues/221 . +see . If a provider hosts one or more advisories with a TLP level of AMBER or RED, then these advisories must be access protected. To check these advisories, authorization can be given via custom headers or certificates. diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index fcf6634d..2831cb49 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -1,4 +1,5 @@ ## csaf_downloader + A tool to download CSAF documents from CSAF providers. ### Usage @@ -39,6 +40,8 @@ Help Options: Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider. +If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf-distribution/version` is sent. + If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and downloading procedes from there. Increasing the number of workers opens more connections to the web servers @@ -47,6 +50,7 @@ However, since this also increases the load on the servers, their administrators have taken countermeasures to limit this. If no config file is explictly given the follwing places are searched for a config file: + ``` ~/.config/csaf/downloader.toml ~/.csaf_downloader.toml @@ -56,6 +60,7 @@ csaf_downloader.toml with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems. Supported options in config files: + ``` # directory # not set by default insecure = false @@ -90,6 +95,7 @@ option. E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain the sub strings **white** or **red**. In the config file this has to be noted as: + ``` ignorepattern = [".*white.*", ".*red.*"] ``` @@ -106,16 +112,18 @@ into a given intervall. There are three possible notations: and 'y' for years are recognized. In these cases only integer values are accepted without any fractions. Some examples: + - `"3h"` means downloading the advisories that have changed in the last three hours. - - `"30m"` .. changed within the last thirty minutes. + - `"30m"` .. changed within the last thirty minutes. - `"3M2m"` .. changed within the last three months and two minutes. - - `"2y"` .. changed within the last two years. + - `"2y"` .. changed within the last two years. 2. Absolute. If the given string is an RFC 3339 date timestamp the time interval between this date and now is used. E.g. `"2006-01-02"` means that all files between 2006 January 2nd and now going to being downloaded. Accepted patterns are: + - `"2006-01-02T15:04:05Z"` - `"2006-01-02T15:04:05+07:00"` - `"2006-01-02T15:04:05-07:00"` @@ -134,6 +142,7 @@ into a given intervall. There are three possible notations: All interval boundaries are inclusive. #### Forwarding + The downloader is able to forward downloaded advisories and their checksums, OpenPGP signatures and validation results to an HTTP endpoint. The details of the implemented API are described [here](https://github.com/mfd2007/csaf_upload_interface). From 1a2ce684ff94a0f47a4b9737698b1961b4aae91b Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Thu, 1 Aug 2024 14:53:23 +0200 Subject: [PATCH 3/4] improve default header * use `csaf_distribution` with an underscore as underscores are allowed by RFC9110 and it is more consistent as it is used with underscore at other places. * change example to `VERSION` to indicate that this is dynamic. --- docs/csaf_checker.md | 2 +- docs/csaf_downloader.md | 2 +- util/client.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index a5bc0bf1..0b223b68 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -30,7 +30,7 @@ Help Options: Will check all given _domains_, by trying each as a CSAF provider. -If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf-distribution/version` is sent. +If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent. If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and checking proceeds from there. diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index 2831cb49..003ae4a8 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -40,7 +40,7 @@ Help Options: Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider. -If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf-distribution/version` is sent. +If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent. If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and downloading procedes from there. diff --git a/util/client.go b/util/client.go index 441aaaa1..b4478ca2 100644 --- a/util/client.go +++ b/util/client.go @@ -64,7 +64,7 @@ func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) { // Use default user agent if none is set if userAgent := hc.Header.Get("User-Agent"); userAgent == "" { - req.Header.Add("User-Agent", "csaf-distribution/"+SemVersion) + req.Header.Add("User-Agent", "csaf_distribution/"+SemVersion) } return hc.Client.Do(req) } From 13a635c7e34c56e4ff39cbfc1ef1e2b6e7bd230a Mon Sep 17 00:00:00 2001 From: koplas <54645365+koplas@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:43:35 +0200 Subject: [PATCH 4/4] Add user-agent documentation to aggregator --- docs/csaf_aggregator.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index 36cbe7ea..661871c1 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -16,6 +16,7 @@ Help Options: ``` If no config file is explictly given the follwing places are searched for a config file: + ``` ~/.config/csaf/aggregator.toml ~/.csaf_aggregator.toml @@ -25,6 +26,7 @@ csaf_aggregator.toml with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems. Usage example for a single run, to test if the config is good: + ```bash ./csaf_aggregator -c docs/examples/aggregator.toml ``` @@ -62,7 +64,6 @@ SHELL=/bin/bash 30 0-23 * * * $HOME/bin/csaf_aggregator --config /etc/csaf_aggregator.toml --interim >> /var/log/csaf_aggregator/interim.log 2>&1 ``` - #### serve via web server Serve the paths where the aggregator writes its `html/` output @@ -78,7 +79,6 @@ a template. For the aggregator the difference is that you can leave out the cgi-bin part, potentially commend out the TLS client parts and adjust the `root` path accordingly. - ### config options The config file is written in [TOML](https://toml.io/en/v1.0.0). @@ -118,10 +118,12 @@ Next we have two TOML _tables_: aggregator // basic infos for the aggregator object remote_validator // config for optional remote validation checker ``` + [See the provider config](csaf_provider.md#provider-options) about how to configure `remote_validator`. At last there is the TOML _array of tables_: + ``` providers // each entry to be mirrored or listed ``` @@ -148,6 +150,9 @@ header Where valid `name` and `domain` settings are required. +If no user agent is specified with `header = "user-agent:custom-agent/1.0"` +then the default agent in the form of `csaf_distribution/VERSION` is sent. + If you want an entry to be listed instead of mirrored in a `aggregator.category == "aggregator"` instance, set `category` to `lister` in the entry. @@ -165,15 +170,16 @@ To offer an easy way of assorting CSAF documents by criteria like document category, languages or values of the branch category within the product tree, ROLIE category values can be configured in `categories`. This can either -be done using an array of strings taken literally or, by prepending `"expr:"`. -The latter is evaluated as JSONPath and the result will be added into the +be done using an array of strings taken literally or, by prepending `"expr:"`. +The latter is evaluated as JSONPath and the result will be added into the categories document. For a more detailed explanation and examples, [refer to the provider config](csaf_provider.md#provider-options). - #### Example config file + + ```toml workers = 2 folder = "/var/csaf_aggregator" @@ -233,8 +239,8 @@ insecure = true category = "lister" # ignore_pattern = [".*white.*", ".*red.*"] ``` - + #### Publish others' advisories