From 5d414f58e5eda2659f44d17be9ea39da87f9e791 Mon Sep 17 00:00:00 2001 From: blotus Date: Fri, 8 Nov 2024 15:11:09 +0100 Subject: [PATCH] Add explicit configuration for signals sharing and blocklists pull (#3277) --- cmd/crowdsec-cli/clicapi/capi.go | 21 ++++++++++ pkg/apiclient/decisions_service.go | 13 ++++++ pkg/apiclient/decisions_service_test.go | 27 +++++++++++-- pkg/apiserver/apic.go | 54 ++++++++++++++++--------- pkg/apiserver/apic_test.go | 32 ++++++++++++++- pkg/csconfig/api.go | 24 ++++++++++- pkg/csconfig/api_test.go | 5 +++ pkg/modelscapi/centralapi_swagger.yaml | 13 ++++++ 8 files changed, 165 insertions(+), 24 deletions(-) diff --git a/cmd/crowdsec-cli/clicapi/capi.go b/cmd/crowdsec-cli/clicapi/capi.go index cba66f11104..61d59836fdd 100644 --- a/cmd/crowdsec-cli/clicapi/capi.go +++ b/cmd/crowdsec-cli/clicapi/capi.go @@ -225,6 +225,27 @@ func (cli *cliCapi) Status(ctx context.Context, out io.Writer, hub *cwhub.Hub) e fmt.Fprint(out, "Your instance is enrolled in the console\n") } + switch *cfg.API.Server.OnlineClient.Sharing { + case true: + fmt.Fprint(out, "Sharing signals is enabled\n") + case false: + fmt.Fprint(out, "Sharing signals is disabled\n") + } + + switch *cfg.API.Server.OnlineClient.PullConfig.Community { + case true: + fmt.Fprint(out, "Pulling community blocklist is enabled\n") + case false: + fmt.Fprint(out, "Pulling community blocklist is disabled\n") + } + + switch *cfg.API.Server.OnlineClient.PullConfig.Blocklists { + case true: + fmt.Fprint(out, "Pulling blocklists from the console is enabled\n") + case false: + fmt.Fprint(out, "Pulling blocklists from the console is disabled\n") + } + return nil } diff --git a/pkg/apiclient/decisions_service.go b/pkg/apiclient/decisions_service.go index 98f26cad9ae..fea2f39072d 100644 --- a/pkg/apiclient/decisions_service.go +++ b/pkg/apiclient/decisions_service.go @@ -31,6 +31,8 @@ type DecisionsListOpts struct { type DecisionsStreamOpts struct { Startup bool `url:"startup,omitempty"` + CommunityPull bool `url:"community_pull"` + AdditionalPull bool `url:"additional_pull"` Scopes string `url:"scopes,omitempty"` ScenariosContaining string `url:"scenarios_containing,omitempty"` ScenariosNotContaining string `url:"scenarios_not_containing,omitempty"` @@ -43,6 +45,17 @@ func (o *DecisionsStreamOpts) addQueryParamsToURL(url string) (string, error) { return "", err } + //Those 2 are a bit different + //They default to true, and we only want to include them if they are false + + if params.Get("community_pull") == "true" { + params.Del("community_pull") + } + + if params.Get("additional_pull") == "true" { + params.Del("additional_pull") + } + return fmt.Sprintf("%s?%s", url, params.Encode()), nil } diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index 54c44f43eda..942d14689ff 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/url" + "strings" "testing" log "github.com/sirupsen/logrus" @@ -87,7 +88,7 @@ func TestDecisionsStream(t *testing.T) { testMethod(t, r, http.MethodGet) if r.Method == http.MethodGet { - if r.URL.RawQuery == "startup=true" { + if strings.Contains(r.URL.RawQuery, "startup=true") { w.WriteHeader(http.StatusOK) w.Write([]byte(`{"deleted":null,"new":[{"duration":"3h59m55.756182786s","id":4,"origin":"cscli","scenario":"manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'","scope":"Ip","type":"ban","value":"1.2.3.4"}]}`)) } else { @@ -160,7 +161,7 @@ func TestDecisionsStreamV3Compatibility(t *testing.T) { testMethod(t, r, http.MethodGet) if r.Method == http.MethodGet { - if r.URL.RawQuery == "startup=true" { + if strings.Contains(r.URL.RawQuery, "startup=true") { w.WriteHeader(http.StatusOK) w.Write([]byte(`{"deleted":[{"scope":"ip","decisions":["1.2.3.5"]}],"new":[{"scope":"ip", "scenario": "manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'", "decisions":[{"duration":"3h59m55.756182786s","value":"1.2.3.4"}]}]}`)) } else { @@ -429,6 +430,8 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) { Scopes string ScenariosContaining string ScenariosNotContaining string + CommunityPull bool + AdditionalPull bool } tests := []struct { @@ -440,11 +443,17 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) { { name: "no filter", expected: baseURLString + "?", + fields: fields{ + CommunityPull: true, + AdditionalPull: true, + }, }, { name: "startup=true", fields: fields{ - Startup: true, + Startup: true, + CommunityPull: true, + AdditionalPull: true, }, expected: baseURLString + "?startup=true", }, @@ -455,9 +464,19 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) { Scopes: "ip,range", ScenariosContaining: "ssh", ScenariosNotContaining: "bf", + CommunityPull: true, + AdditionalPull: true, }, expected: baseURLString + "?scenarios_containing=ssh&scenarios_not_containing=bf&scopes=ip%2Crange&startup=true", }, + { + name: "pull options", + fields: fields{ + CommunityPull: false, + AdditionalPull: false, + }, + expected: baseURLString + "?additional_pull=false&community_pull=false", + }, } for _, tt := range tests { @@ -467,6 +486,8 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) { Scopes: tt.fields.Scopes, ScenariosContaining: tt.fields.ScenariosContaining, ScenariosNotContaining: tt.fields.ScenariosNotContaining, + CommunityPull: tt.fields.CommunityPull, + AdditionalPull: tt.fields.AdditionalPull, } got, err := o.addQueryParamsToURL(baseURLString) diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index fff0ebcacbf..51a85b1ea23 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -69,6 +69,10 @@ type apic struct { consoleConfig *csconfig.ConsoleConfig isPulling chan bool whitelists *csconfig.CapiWhitelist + + pullBlocklists bool + pullCommunity bool + shareSignals bool } // randomDuration returns a duration value between d-delta and d+delta @@ -198,6 +202,9 @@ func NewAPIC(ctx context.Context, config *csconfig.OnlineApiClientCfg, dbClient usageMetricsIntervalFirst: randomDuration(usageMetricsInterval, usageMetricsIntervalDelta), isPulling: make(chan bool, 1), whitelists: apicWhitelist, + pullBlocklists: *config.PullConfig.Blocklists, + pullCommunity: *config.PullConfig.Community, + shareSignals: *config.Sharing, } password := strfmt.Password(config.Credentials.Password) @@ -295,7 +302,7 @@ func (a *apic) Push(ctx context.Context) error { var signals []*models.AddSignalsRequestItem for _, alert := range alerts { - if ok := shouldShareAlert(alert, a.consoleConfig); ok { + if ok := shouldShareAlert(alert, a.consoleConfig, a.shareSignals); ok { signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert), *a.consoleConfig.ShareContext)) } } @@ -324,7 +331,13 @@ func getScenarioTrustOfAlert(alert *models.Alert) string { return scenarioTrust } -func shouldShareAlert(alert *models.Alert, consoleConfig *csconfig.ConsoleConfig) bool { +func shouldShareAlert(alert *models.Alert, consoleConfig *csconfig.ConsoleConfig, shareSignals bool) bool { + + if !shareSignals { + log.Debugf("sharing signals is disabled") + return false + } + if *alert.Simulated { log.Debugf("simulation enabled for alert (id:%d), will not be sent to CAPI", alert.ID) return false @@ -625,7 +638,9 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { log.Infof("Starting community-blocklist update") - data, _, err := a.apiClient.Decisions.GetStreamV3(ctx, apiclient.DecisionsStreamOpts{Startup: a.startup}) + log.Debugf("Community pull: %t | Blocklist pull: %t", a.pullCommunity, a.pullBlocklists) + + data, _, err := a.apiClient.Decisions.GetStreamV3(ctx, apiclient.DecisionsStreamOpts{Startup: a.startup, CommunityPull: a.pullCommunity, AdditionalPull: a.pullBlocklists}) if err != nil { return fmt.Errorf("get stream: %w", err) } @@ -650,23 +665,26 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { log.Printf("capi/community-blocklist : %d explicit deletions", nbDeleted) - if len(data.New) == 0 { - log.Infof("capi/community-blocklist : received 0 new entries (expected if you just installed crowdsec)") - return nil - } + if len(data.New) > 0 { + // create one alert for community blocklist using the first decision + decisions := a.apiClient.Decisions.GetDecisionsFromGroups(data.New) + // apply APIC specific whitelists + decisions = a.ApplyApicWhitelists(decisions) - // create one alert for community blocklist using the first decision - decisions := a.apiClient.Decisions.GetDecisionsFromGroups(data.New) - // apply APIC specific whitelists - decisions = a.ApplyApicWhitelists(decisions) + alert := createAlertForDecision(decisions[0]) + alertsFromCapi := []*models.Alert{alert} + alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, addCounters) - alert := createAlertForDecision(decisions[0]) - alertsFromCapi := []*models.Alert{alert} - alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, addCounters) - - err = a.SaveAlerts(ctx, alertsFromCapi, addCounters, deleteCounters) - if err != nil { - return fmt.Errorf("while saving alerts: %w", err) + err = a.SaveAlerts(ctx, alertsFromCapi, addCounters, deleteCounters) + if err != nil { + return fmt.Errorf("while saving alerts: %w", err) + } + } else { + if a.pullCommunity { + log.Info("capi/community-blocklist : received 0 new entries (expected if you just installed crowdsec)") + } else { + log.Debug("capi/community-blocklist : community blocklist pull is disabled") + } } // update blocklists diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index 99fee6e32bf..a8fbb40c4fa 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -69,7 +69,10 @@ func getAPIC(t *testing.T, ctx context.Context) *apic { ShareCustomScenarios: ptr.Of(false), ShareContext: ptr.Of(false), }, - isPulling: make(chan bool, 1), + isPulling: make(chan bool, 1), + shareSignals: true, + pullBlocklists: true, + pullCommunity: true, } } @@ -200,6 +203,11 @@ func TestNewAPIC(t *testing.T) { Login: "foo", Password: "bar", }, + Sharing: ptr.Of(true), + PullConfig: csconfig.CapiPullConfig{ + Community: ptr.Of(true), + Blocklists: ptr.Of(true), + }, } } @@ -1193,6 +1201,7 @@ func TestShouldShareAlert(t *testing.T) { tests := []struct { name string consoleConfig *csconfig.ConsoleConfig + shareSignals bool alert *models.Alert expectedRet bool expectedTrust string @@ -1203,6 +1212,7 @@ func TestShouldShareAlert(t *testing.T) { ShareCustomScenarios: ptr.Of(true), }, alert: &models.Alert{Simulated: ptr.Of(false)}, + shareSignals: true, expectedRet: true, expectedTrust: "custom", }, @@ -1212,6 +1222,7 @@ func TestShouldShareAlert(t *testing.T) { ShareCustomScenarios: ptr.Of(false), }, alert: &models.Alert{Simulated: ptr.Of(false)}, + shareSignals: true, expectedRet: false, expectedTrust: "custom", }, @@ -1220,6 +1231,7 @@ func TestShouldShareAlert(t *testing.T) { consoleConfig: &csconfig.ConsoleConfig{ ShareManualDecisions: ptr.Of(true), }, + shareSignals: true, alert: &models.Alert{ Simulated: ptr.Of(false), Decisions: []*models.Decision{{Origin: ptr.Of(types.CscliOrigin)}}, @@ -1232,6 +1244,7 @@ func TestShouldShareAlert(t *testing.T) { consoleConfig: &csconfig.ConsoleConfig{ ShareManualDecisions: ptr.Of(false), }, + shareSignals: true, alert: &models.Alert{ Simulated: ptr.Of(false), Decisions: []*models.Decision{{Origin: ptr.Of(types.CscliOrigin)}}, @@ -1244,6 +1257,7 @@ func TestShouldShareAlert(t *testing.T) { consoleConfig: &csconfig.ConsoleConfig{ ShareTaintedScenarios: ptr.Of(true), }, + shareSignals: true, alert: &models.Alert{ Simulated: ptr.Of(false), ScenarioHash: ptr.Of("whateverHash"), @@ -1256,6 +1270,7 @@ func TestShouldShareAlert(t *testing.T) { consoleConfig: &csconfig.ConsoleConfig{ ShareTaintedScenarios: ptr.Of(false), }, + shareSignals: true, alert: &models.Alert{ Simulated: ptr.Of(false), ScenarioHash: ptr.Of("whateverHash"), @@ -1263,11 +1278,24 @@ func TestShouldShareAlert(t *testing.T) { expectedRet: false, expectedTrust: "tainted", }, + { + name: "manual alert should not be shared if global sharing is disabled", + consoleConfig: &csconfig.ConsoleConfig{ + ShareManualDecisions: ptr.Of(true), + }, + shareSignals: false, + alert: &models.Alert{ + Simulated: ptr.Of(false), + ScenarioHash: ptr.Of("whateverHash"), + }, + expectedRet: false, + expectedTrust: "manual", + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - ret := shouldShareAlert(tc.alert, tc.consoleConfig) + ret := shouldShareAlert(tc.alert, tc.consoleConfig, tc.shareSignals) assert.Equal(t, tc.expectedRet, ret) }) } diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index 3014b729a9e..5f2f8f9248b 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -38,10 +38,17 @@ type ApiCredentialsCfg struct { CertPath string `yaml:"cert_path,omitempty"` } -/*global api config (for lapi->oapi)*/ +type CapiPullConfig struct { + Community *bool `yaml:"community,omitempty"` + Blocklists *bool `yaml:"blocklists,omitempty"` +} + +/*global api config (for lapi->capi)*/ type OnlineApiClientCfg struct { CredentialsFilePath string `yaml:"credentials_path,omitempty"` // credz will be edited by software, store in diff file Credentials *ApiCredentialsCfg `yaml:"-"` + PullConfig CapiPullConfig `yaml:"pull,omitempty"` + Sharing *bool `yaml:"sharing,omitempty"` } /*local api config (for crowdsec/cscli->lapi)*/ @@ -344,6 +351,21 @@ func (c *Config) LoadAPIServer(inCli bool) error { log.Printf("push and pull to Central API disabled") } + //Set default values for CAPI push/pull + if c.API.Server.OnlineClient != nil { + if c.API.Server.OnlineClient.PullConfig.Community == nil { + c.API.Server.OnlineClient.PullConfig.Community = ptr.Of(true) + } + + if c.API.Server.OnlineClient.PullConfig.Blocklists == nil { + c.API.Server.OnlineClient.PullConfig.Blocklists = ptr.Of(true) + } + + if c.API.Server.OnlineClient.Sharing == nil { + c.API.Server.OnlineClient.Sharing = ptr.Of(true) + } + } + if err := c.LoadDBConfig(inCli); err != nil { return err } diff --git a/pkg/csconfig/api_test.go b/pkg/csconfig/api_test.go index dff3c3afc8c..17802ba31dd 100644 --- a/pkg/csconfig/api_test.go +++ b/pkg/csconfig/api_test.go @@ -212,6 +212,11 @@ func TestLoadAPIServer(t *testing.T) { Login: "test", Password: "testpassword", }, + Sharing: ptr.Of(true), + PullConfig: CapiPullConfig{ + Community: ptr.Of(true), + Blocklists: ptr.Of(true), + }, }, Profiles: tmpLAPI.Profiles, ProfilesPath: "./testdata/profiles.yaml", diff --git a/pkg/modelscapi/centralapi_swagger.yaml b/pkg/modelscapi/centralapi_swagger.yaml index bd695894f2b..c75233809c8 100644 --- a/pkg/modelscapi/centralapi_swagger.yaml +++ b/pkg/modelscapi/centralapi_swagger.yaml @@ -55,6 +55,19 @@ paths: description: "returns list of top decisions to add or delete" produces: - "application/json" + parameters: + - in: query + name: "community_pull" + type: "boolean" + default: true + required: false + description: "Fetch the community blocklist content" + - in: query + name: "additional_pull" + type: "boolean" + default: true + required: false + description: "Fetch additional blocklists content" responses: "200": description: "200 response"