From 791901fb83d2b9202bf343c95a54b56f44afb733 Mon Sep 17 00:00:00 2001 From: he2ss Date: Wed, 23 Oct 2024 15:14:58 +0200 Subject: [PATCH 01/30] add HTTP datasource --- pkg/acquisition/modules/http/http.go | 353 +++++++++ pkg/acquisition/modules/http/http_test.go | 702 ++++++++++++++++++ pkg/acquisition/modules/http/testdata/ca.crt | 23 + .../modules/http/testdata/client.crt | 24 + .../modules/http/testdata/client.key | 27 + .../modules/http/testdata/server.crt | 23 + .../modules/http/testdata/server.key | 27 + 7 files changed, 1179 insertions(+) create mode 100644 pkg/acquisition/modules/http/http.go create mode 100644 pkg/acquisition/modules/http/http_test.go create mode 100644 pkg/acquisition/modules/http/testdata/ca.crt create mode 100644 pkg/acquisition/modules/http/testdata/client.crt create mode 100644 pkg/acquisition/modules/http/testdata/client.key create mode 100644 pkg/acquisition/modules/http/testdata/server.crt create mode 100644 pkg/acquisition/modules/http/testdata/server.key diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go new file mode 100644 index 00000000000..c7347feb772 --- /dev/null +++ b/pkg/acquisition/modules/http/http.go @@ -0,0 +1,353 @@ +package httpacquisition + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "os" + "time" + + // import gin for http server + + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + + "gopkg.in/tomb.v2" + "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" +) + +var ( + dataSourceName = "http" +) + +var linesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_httpsource_hits_total", + Help: "Total lines that were read from http source", + }, + []string{"path"}) + +const () + +type HttpConfiguration struct { + //IPFilter []string `yaml:"ip_filter"` + Port int `yaml:"port"` + Path string `yaml:"path"` + AuthType string `yaml:"auth_type"` + BasicAuth *BasicAuthConfig `yaml:"basic_auth"` + Headers *map[string]string `yaml:"headers"` + TLS *TLSConfig `yaml:"tls"` + CustomStatusCode *int `yaml:"custom_status_code"` + CustomHeaders *map[string]string `yaml:"custom_headers"` + MaxBodySize *int64 `yaml:"max_body_size"` + ChunkSize *int64 `yaml:"chunk_size"` + Timeout *time.Duration `yaml:"timeout"` + configuration.DataSourceCommonCfg `yaml:",inline"` +} + +type BasicAuthConfig struct { + Username string `yaml:"username"` + Password string `yaml:"password"` +} + +type TLSConfig struct { + InsecureSkipVerify bool `yaml:"insecure_skip_verify"` + ServerCert string `yaml:"server_cert"` + ServerKey string `yaml:"server_key"` + CaCert string `yaml:"ca_cert"` +} + +type HTTPSource struct { + Config HttpConfiguration + logger *log.Entry + Server *http.Server +} + +func (h *HTTPSource) GetUuid() string { + return h.Config.UniqueId +} + +func (h *HTTPSource) UnmarshalConfig(yamlConfig []byte) error { + h.Config = HttpConfiguration{} + err := yaml.Unmarshal(yamlConfig, &h.Config) + if err != nil { + return fmt.Errorf("cannot parse %s datasource configuration: %w", dataSourceName, err) + } + return nil +} + +func (hc *HttpConfiguration) Validate() error { + if hc.Port == 0 { + return fmt.Errorf("port is required") + } + if hc.Path == "" { + return fmt.Errorf("path is required") + } + + switch hc.AuthType { + case "basic_auth": + if hc.BasicAuth == nil { + return fmt.Errorf("basic_auth is required") + } + if hc.BasicAuth.Username == "" { + return fmt.Errorf("username is required") + } + if hc.BasicAuth.Password == "" { + return fmt.Errorf("password is required") + } + case "headers": + if hc.Headers == nil { + return fmt.Errorf("headers is required") + } + case "mtls": + if hc.TLS == nil || hc.TLS != nil && hc.TLS.CaCert == "" { + return fmt.Errorf("ca_cert is required") + } + default: + if hc.TLS == nil { + return fmt.Errorf("at least one of tls or auth_type is required") + } + } + + if hc.TLS != nil { + if hc.TLS.ServerCert == "" { + return fmt.Errorf("server_cert is required") + } + if hc.TLS.ServerKey == "" { + return fmt.Errorf("server_key is required") + } + } + + if hc.MaxBodySize != nil && *hc.MaxBodySize <= 0 { + return fmt.Errorf("max_body_size must be positive") + } + + if hc.ChunkSize != nil && *hc.ChunkSize <= 0 { + return fmt.Errorf("chunk_size must be positive") + } + + // validate custom status code + if hc.CustomStatusCode != nil { + statusText := http.StatusText(*hc.CustomStatusCode) + if statusText == "" { + return fmt.Errorf("invalid HTTP status code") + } + } + + return nil +} + +func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry) error { + h.logger = logger + err := h.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + + if err := h.Config.Validate(); err != nil { + return fmt.Errorf("invalid configuration: %w", err) + } + + return nil +} + +func (h *HTTPSource) ConfigureByDSN(string, map[string]string, *log.Entry, string) error { + return fmt.Errorf("%s datasource does not support command-line acquisition", dataSourceName) +} + +func (h *HTTPSource) GetMode() string { + return h.Config.Mode +} + +func (h *HTTPSource) GetName() string { + return dataSourceName +} + +func (k *HTTPSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { + return fmt.Errorf("%s datasource does not support one-shot acquisition", dataSourceName) +} + +func (h *HTTPSource) CanRun() error { + return nil +} + +func (h *HTTPSource) GetMetrics() []prometheus.Collector { + return []prometheus.Collector{linesRead} +} + +func (k *HTTPSource) GetAggregMetrics() []prometheus.Collector { + return []prometheus.Collector{linesRead} +} + +func (h *HTTPSource) Dump() interface{} { + return h +} + +func (hc *HttpConfiguration) NewTLSConfig() (*tls.Config, error) { + tlsConfig := tls.Config{ + InsecureSkipVerify: hc.TLS.InsecureSkipVerify, + } + + if hc.TLS.ServerCert != "" && hc.TLS.ServerKey != "" { + cert, err := tls.LoadX509KeyPair(hc.TLS.ServerCert, hc.TLS.ServerKey) + if err != nil { + return nil, fmt.Errorf("failed to load server cert/key: %w", err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + } + + if hc.AuthType == "mtls" && hc.TLS.CaCert != "" { + caCert, err := os.ReadFile(hc.TLS.CaCert) + if err != nil { + return nil, fmt.Errorf("failed to read ca cert: %w", err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.ClientCAs = caCertPool + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + } + + return &tlsConfig, nil +} + +func authorizeRequest(r *http.Request, hc *HttpConfiguration) error { + if hc.AuthType == "basic_auth" { + username, password, ok := r.BasicAuth() + if !ok { + return fmt.Errorf("missing basic auth") + } + if username != hc.BasicAuth.Username || password != hc.BasicAuth.Password { + return fmt.Errorf("invalid basic auth") + } + } + if hc.AuthType == "headers" { + for key, value := range *hc.Headers { + if r.Header.Get(key) != value { + return fmt.Errorf("invalid headers") + } + } + } + return nil +} + +func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc *HttpConfiguration, out chan types.Event, t *tomb.Tomb) error { + if hc.MaxBodySize != nil && r.ContentLength > *hc.MaxBodySize { + w.WriteHeader(http.StatusRequestEntityTooLarge) + return fmt.Errorf("body size exceeds max body size: %d > %d", r.ContentLength, *hc.MaxBodySize) + } + body, err := io.ReadAll(r.Body) + defer r.Body.Close() + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return fmt.Errorf("failed to read body: %w", err) + } + h.logger.Tracef("body received: %+v", string(body)) + + t.Go(func() error { + line := types.Line{ + Raw: string(body), + Src: r.RemoteAddr, + Time: time.Now().UTC(), + Labels: hc.Labels, + Process: true, + Module: h.GetName(), + } + + evt := types.Event{ + Line: line, + Process: true, + Type: types.LOG, + ExpectMode: types.LIVE, + } + out <- evt + return nil + }) + + return nil +} + +func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { + mux := http.NewServeMux() + mux.HandleFunc(h.Config.Path, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + h.logger.Errorf("method not allowed: %s", r.Method) + http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + if err := authorizeRequest(r, &h.Config); err != nil { + h.logger.Errorf("failed to authorize request: %s", err) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + err := h.processRequest(w, r, &h.Config, out, t) + if err != nil { + h.logger.Errorf("failed to process request: %s", err) + return + } + + if h.Config.CustomHeaders != nil { + for key, value := range *h.Config.CustomHeaders { + w.Header().Set(key, value) + } + } + if h.Config.CustomStatusCode != nil { + w.WriteHeader(*h.Config.CustomStatusCode) + } else { + w.WriteHeader(http.StatusOK) + } + + w.Write([]byte("OK")) + }) + + h.Server = &http.Server{ + Addr: fmt.Sprintf(":%d", h.Config.Port), + Handler: mux, + } + + if h.Config.Timeout != nil { + h.Server.ReadTimeout = *h.Config.Timeout + } + + if h.Config.TLS != nil { + tlsConfig, err := h.Config.NewTLSConfig() + if err != nil { + return fmt.Errorf("failed to create tls config: %w", err) + } + h.logger.Tracef("tls config: %+v", tlsConfig) + h.Server.TLSConfig = tlsConfig + } + + if h.Config.TLS != nil { + h.logger.Infof("start https server on port %d", h.Config.Port) + err := h.Server.ListenAndServeTLS(h.Config.TLS.ServerCert, h.Config.TLS.ServerKey) + if err != nil && err != http.ErrServerClosed { + return fmt.Errorf("https server failed: %w", err) + } + } else { + h.logger.Infof("start http server on port %d", h.Config.Port) + err := h.Server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + return fmt.Errorf("http server failed: %w", err) + } + } + + return nil +} + +func (h *HTTPSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { + h.logger.Debugf("start http server on port %d", h.Config.Port) + + t.Go(func() error { + defer trace.CatchPanic("crowdsec/acquis/http/live") + return h.RunServer(out, t) + }) + + return nil +} diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go new file mode 100644 index 00000000000..3752496aefc --- /dev/null +++ b/pkg/acquisition/modules/http/http_test.go @@ -0,0 +1,702 @@ +package httpacquisition + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/pkg/cstest" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" +) + +const ( + testHTTPServerAddr = "http://127.0.0.1:8080" + testHTTPServerAddrTLS = "https://127.0.0.1:8080" +) + +func TestConfigure(t *testing.T) { + tests := []struct { + config string + expectedErr string + }{ + { + config: ` +foobar: bla`, + expectedErr: "invalid configuration: port is required", + }, + { + config: ` +source: http +port: aa`, + expectedErr: "cannot parse http datasource configuration: yaml: unmarshal errors:\n line 3: cannot unmarshal !!str `aa` into int", + }, + { + config: ` +source: http +port: 8080`, + expectedErr: "invalid configuration: path is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: basic_auth`, + expectedErr: "invalid configuration: basic_auth is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers`, + expectedErr: "invalid configuration: headers is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: basic_auth +basic_auth: + username: 132`, + expectedErr: "invalid configuration: password is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: basic_auth +basic_auth: + password: 132`, + expectedErr: "invalid configuration: username is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers +headers:`, + expectedErr: "invalid configuration: headers is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: toto`, + expectedErr: "invalid configuration: at least one of tls or auth_type is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: value +tls: + server_key: key`, + expectedErr: "invalid configuration: server_cert is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: value +tls: + server_cert: cert`, + expectedErr: "invalid configuration: server_key is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: mtls +tls: + server_cert: cert + server_key: key`, + expectedErr: "invalid configuration: ca_cert is required", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: value +max_body_size: 0`, + expectedErr: "invalid configuration: max_body_size must be positive", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: value +chunk_size: 0`, + expectedErr: "invalid configuration: chunk_size must be positive", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: value +timeout: toto`, + expectedErr: "cannot parse http datasource configuration: yaml: unmarshal errors:\n line 8: cannot unmarshal !!str `toto` into time.Duration", + }, + { + config: ` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: value +custom_status_code: 999`, + expectedErr: "invalid configuration: invalid HTTP status code", + }, + } + + subLogger := log.WithFields(log.Fields{ + "type": "http", + }) + + for _, test := range tests { + h := HTTPSource{} + err := h.Configure([]byte(test.config), subLogger) + cstest.AssertErrorContains(t, err, test.expectedErr) + } +} + +func TestGetUuid(t *testing.T) { + h := HTTPSource{} + h.Config.UniqueId = "test" + if h.GetUuid() != "test" { + t.Fatalf("expected 'test', got '%s'", h.GetUuid()) + } +} + +func TestUnmarshalConfig(t *testing.T) { + h := HTTPSource{} + err := h.UnmarshalConfig([]byte(` +source: http +port: 8080 +path: 15 + auth_type: headers`)) + cstest.AssertErrorMessage(t, err, "cannot parse http datasource configuration: yaml: line 5: found a tab character that violates indentation") +} + +func TestConfigureByDSN(t *testing.T) { + h := HTTPSource{} + err := h.ConfigureByDSN("http://localhost:8080/test", map[string]string{}, log.WithFields(log.Fields{ + "type": "http", + }), "test") + cstest.AssertErrorMessage( + t, + err, + "http datasource does not support command-line acquisition", + ) +} + +func TestGetMode(t *testing.T) { + h := HTTPSource{} + h.Config.Mode = "test" + if h.GetMode() != "test" { + t.Fatalf("expected 'test', got '%s'", h.GetMode()) + } +} + +func TestGetName(t *testing.T) { + h := HTTPSource{} + if h.GetName() != "http" { + t.Fatalf("expected 'http', got '%s'", h.GetName()) + } +} + +func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte) (chan types.Event, *tomb.Tomb) { + subLogger := log.WithFields(log.Fields{ + "type": "http", + }) + err := h.Configure(config, subLogger) + if err != nil { + t.Fatalf("unable to configure http source: %s", err) + } + tomb := tomb.Tomb{} + out := make(chan types.Event) + err = h.StreamingAcquisition(out, &tomb) + if err != nil { + t.Fatalf("unable to start streaming acquisition: %s", err) + } + return out, &tomb +} + +func TestStreamingAcquisitionWrongHTTPMethod(t *testing.T) { + h := &HTTPSource{} + _, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: basic_auth +basic_auth: + username: test + password: test`)) + + time.Sleep(1 * time.Second) + + res, err := http.Get(fmt.Sprintf("%s/test", testHTTPServerAddr)) + if err != nil { + t.Fatalf("unable to get http response: %s", err) + } + if res.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("expected status code %d, got %d", http.StatusMethodNotAllowed, res.StatusCode) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() + +} + +func TestStreamingAcquisitionUnknownPath(t *testing.T) { + h := &HTTPSource{} + _, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: basic_auth +basic_auth: + username: test + password: test`)) + + time.Sleep(1 * time.Second) + + res, err := http.Get(fmt.Sprintf("%s/unknown", testHTTPServerAddr)) + if err != nil { + t.Fatalf("unable to get http response: %s", err) + } + + if res.StatusCode != http.StatusNotFound { + t.Fatalf("expected status code %d, got %d", http.StatusNotFound, res.StatusCode) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionBasicAuth(t *testing.T) { + h := &HTTPSource{} + _, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: basic_auth +basic_auth: + username: test + password: test`)) + + time.Sleep(1 * time.Second) + + client := &http.Client{} + + resp, err := http.Post(fmt.Sprintf("%s/test", testHTTPServerAddr), "application/json", strings.NewReader("test")) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected status code %d, got %d", http.StatusUnauthorized, resp.StatusCode) + } + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader("test")) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.SetBasicAuth("test", "WrongPassword") + resp, err = client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected status code %d, got %d", http.StatusUnauthorized, resp.StatusCode) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionBadHeaders(t *testing.T) { + h := &HTTPSource{} + _, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test`)) + + time.Sleep(1 * time.Second) + + client := &http.Client{} + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader("test")) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.Header.Add("key", "wrong") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected status code %d, got %d", http.StatusUnauthorized, resp.StatusCode) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionMaxBodySize(t *testing.T) { + h := &HTTPSource{} + _, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test +max_body_size: 5`)) + + time.Sleep(1 * time.Second) + + client := &http.Client{} + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader("testtest")) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.Header.Add("key", "test") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("expected status code %d, got %d", http.StatusRequestEntityTooLarge, resp.StatusCode) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionSuccess(t *testing.T) { + h := &HTTPSource{} + out, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test`)) + + time.Sleep(1 * time.Second) + rawEvt := `{"test": "test"}` + + client := &http.Client{} + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.Header.Add("key", "test") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + assertEvents(out, t, rawEvt) + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionCustomStatusCodeAndCustomHeaders(t *testing.T) { + h := &HTTPSource{} + out, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test +custom_status_code: 201 +custom_headers: + success: true`)) + + time.Sleep(1 * time.Second) + + rawEvt := `{"test": "test"}` + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.Header.Add("key", "test") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + + if resp.StatusCode != http.StatusCreated { + t.Fatalf("expected status code %d, got %d", http.StatusCreated, resp.StatusCode) + } + + if resp.Header.Get("success") != "true" { + t.Fatalf("expected header 'success' to be 'true', got '%s'", resp.Header.Get("success")) + } + + assertEvents(out, t, rawEvt) + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +type slowReader struct { + delay time.Duration + body []byte + index int +} + +func (sr *slowReader) Read(p []byte) (int, error) { + if sr.index >= len(sr.body) { + return 0, io.EOF + } + time.Sleep(sr.delay) // Simulate a delay in reading + n := copy(p, sr.body[sr.index:]) + sr.index += n + return n, nil +} + +func assertEvents(out chan types.Event, t *testing.T, expected string) { + readLines := []types.Event{} + + select { + case event := <-out: + readLines = append(readLines, event) + case <-time.After(2 * time.Second): + break + } + + if len(readLines) != 1 { + t.Fatalf("expected 1 line, got %d", len(readLines)) + } + if readLines[0].Line.Raw != expected { + t.Fatalf(`expected %s, got '%+v'`, expected, readLines[0].Line) + } +} + +func TestStreamingAcquisitionTimeout(t *testing.T) { + h := &HTTPSource{} + _, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test +timeout: 1s`)) + + time.Sleep(1 * time.Second) + + slow := &slowReader{ + delay: 2 * time.Second, + body: []byte(`{"test": "delayed_payload"}`), + } + + req, err := http.NewRequest("POST", fmt.Sprintf("%s/test", testHTTPServerAddr), slow) + if err != nil { + t.Fatalf("Error creating request: %v", err) + } + req.Header.Add("key", "test") + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Error sending request: %v", err) + } + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status code %d, got %d", http.StatusBadRequest, resp.StatusCode) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionTLSHTTPRequest(t *testing.T) { + h := &HTTPSource{} + _, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +tls: + server_cert: testdata/server.crt + server_key: testdata/server.key + ca_cert: testdata/ca.crt`)) + + time.Sleep(1 * time.Second) + + resp, err := http.Post(fmt.Sprintf("%s/test", testHTTPServerAddr), "application/json", strings.NewReader("test")) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected status code %d, got %d", http.StatusBadRequest, resp.StatusCode) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionTLSWithHeadersAuthSuccess(t *testing.T) { + h := &HTTPSource{} + out, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test +tls: + server_cert: testdata/server.crt + server_key: testdata/server.key +`)) + + time.Sleep(1 * time.Second) + + caCert, err := os.ReadFile("testdata/server.crt") + if err != nil { + t.Fatalf("unable to read ca cert: %s", err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + tlsConfig := &tls.Config{ + RootCAs: caCertPool, + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } + + rawEvt := `{"test": "test"}` + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.Header.Add("key", "test") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + assertEvents(out, t, rawEvt) + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionMTLS(t *testing.T) { + h := &HTTPSource{} + out, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: mtls +tls: + server_cert: testdata/server.crt + server_key: testdata/server.key + ca_cert: testdata/ca.crt`)) + + time.Sleep(1 * time.Second) + + // init client cert + cert, err := tls.LoadX509KeyPair("testdata/client.crt", "testdata/client.key") + if err != nil { + t.Fatalf("unable to load client cert: %s", err) + } + + caCert, err := os.ReadFile("testdata/ca.crt") + if err != nil { + t.Fatalf("unable to read ca cert: %s", err) + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } + + rawEvt := `{"test": "test"}` + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + assertEvents(out, t, rawEvt) + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} diff --git a/pkg/acquisition/modules/http/testdata/ca.crt b/pkg/acquisition/modules/http/testdata/ca.crt new file mode 100644 index 00000000000..ac81b9db8a6 --- /dev/null +++ b/pkg/acquisition/modules/http/testdata/ca.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDvzCCAqegAwIBAgIUHQfsFpWkCy7gAmDa3A6O+y5CvAswDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCRlIxFjAUBgNVBAgTDUlsZS1kZS1GcmFuY2UxDjAMBgNV +BAcTBVBhcmlzMREwDwYDVQQKEwhDcm93ZHNlYzERMA8GA1UECxMIQ3Jvd2RzZWMx +EjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0yNDEwMjMxMDAxMDBaFw0yOTEwMjIxMDAx +MDBaMG8xCzAJBgNVBAYTAkZSMRYwFAYDVQQIEw1JbGUtZGUtRnJhbmNlMQ4wDAYD +VQQHEwVQYXJpczERMA8GA1UEChMIQ3Jvd2RzZWMxETAPBgNVBAsTCENyb3dkc2Vj +MRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCZSR2/A24bpVHSiEeSlelfdA32uhk9wHkauwy2qxos/G/UmKG/dgWrHzRh +LawlFVHtVn4u7Hjqz2y2EsH3bX42jC5NMVARgXIOBr1dE6F5/bPqA6SoVgkDm9wh +ZBigyAMxYsR4+3ahuf0pQflBShKrLZ1UYoe6tQXob7l3x5vThEhNkBawBkLfWpj7 +7Imm1tGyEZdxCMkT400KRtSmJRrnpiOCUosnacwgp7MCbKWOIOng07Eh16cVUiuI +BthWU/LycIuac2xaD9PFpeK/MpwASRRPXZgPUhiZuaa7vttD0phCdDaS46Oln5/7 +tFRZH0alBZmkpVZJCWAP4ujIA3vLAgMBAAGjUzBRMA4GA1UdDwEB/wQEAwIBBjAP +BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTwpg+WN1nZJs4gj5hfoa+fMSZjGTAP +BgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQAZuOWT8zHcwbWvC6Jm +/ccgB/U7SbeIYFJrCZd9mTyqsgnkFNH8yJ5F4dXXtPXr+SO/uWWa3G5hams3qVFf +zWzzPDQdyhUhfh5fjUHR2RsSGBmCxcapYHpVvAP5aY1/ujYrXMvAJV0hfDO2tGHb +rveuJxhe8ymQ1Yb2u9NcmI1HG9IVt3Airz4gAIUJWbFvRigky0bukfddOkfiUiaF +DMPJQO6HAj8d8ctSHHVZWzhAInZ1pDg6HIHYF44m1tT27pSQoi0ZFocskDi/fC2f +EIF0nu5fRLUS6BZEfpnDi9U0lbJ/kUrgT5IFHMFqXdRpDqcnXpJZhYtp5l6GoqjM +gT33 +-----END CERTIFICATE----- diff --git a/pkg/acquisition/modules/http/testdata/client.crt b/pkg/acquisition/modules/http/testdata/client.crt new file mode 100644 index 00000000000..55efdddad09 --- /dev/null +++ b/pkg/acquisition/modules/http/testdata/client.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID7jCCAtagAwIBAgIUJMTPh3oPJLPgsnb9T85ieb4EuOQwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCRlIxFjAUBgNVBAgTDUlsZS1kZS1GcmFuY2UxDjAMBgNV +BAcTBVBhcmlzMREwDwYDVQQKEwhDcm93ZHNlYzERMA8GA1UECxMIQ3Jvd2RzZWMx +EjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0yNDEwMjMxMDQ2MDBaFw0yNTEwMjMxMDQ2 +MDBaMHIxCzAJBgNVBAYTAkZSMRYwFAYDVQQIEw1JbGUtZGUtRnJhbmNlMQ4wDAYD +VQQHEwVQYXJpczERMA8GA1UEChMIQ3Jvd2RzZWMxFzAVBgNVBAsTDlRlc3Rpbmcg +Y2xpZW50MQ8wDQYDVQQDEwZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDAUOdpRieRrrH6krUjgcjLgJg6TzoWAb/iv6rfcioX1L9bj9fZSkwu +GqKzXX/PceIXElzQgiGJZErbJtnTzhGS80QgtAB8BwWQIT2zgoGcYJf7pPFvmcMM +qMGFwK0dMC+LHPk+ePtFz8dskI2XJ8jgBdtuZcnDblMuVGtjYT6n0rszvRdo118+ +mlGCLPzOfsO1JdOqLWAR88yZfqCFt1TrwmzpRT1crJQeM6i7muw4aO0L7uSek9QM +6APHz0QexSq7/zHOtRjA4jnJbDzZJHRlwOdlsNU9cmTz6uWIQXlg+2ovD55YurNy ++jYfmfDYpimhoeGf54zaETp1fTuTJYpxAgMBAAGjfzB9MA4GA1UdDwEB/wQEAwIF +oDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAd +BgNVHQ4EFgQUmH0/7RuKnoW7sEK4Cr8eVNGbb8swHwYDVR0jBBgwFoAU8KYPljdZ +2SbOII+YX6GvnzEmYxkwDQYJKoZIhvcNAQELBQADggEBAHVn9Zuoyxu9iTFoyJ50 +e/XKcmt2uK2M1x+ap2Av7Wb/Omikx/R2YPq7994BfiUCAezY2YtreZzkE6Io1wNM +qApijEJnlqEmOXiYJqlF89QrCcsAsz6lfaqitYBZSL3o4KT+7/uUDVxgNEjEksRz +9qy6DFBLvyhxbOM2zDEV+MVfemBWSvNiojHqXzDBkZnBHHclJLuIKsXDZDGhKbNd +hsoGU00RLevvcUpUJ3a68ekgwiYFJifm0uyfmao9lmiB3i+8ZW3Q4rbwHtD+U7U2 +3n+U5PkhiUAveuMfrvUMzsTolZiop9ZLtcALDUFaqyr4tjfVOf5+CGjiluio7oE1 +UYg= +-----END CERTIFICATE----- diff --git a/pkg/acquisition/modules/http/testdata/client.key b/pkg/acquisition/modules/http/testdata/client.key new file mode 100644 index 00000000000..f8ef2efbd58 --- /dev/null +++ b/pkg/acquisition/modules/http/testdata/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAwFDnaUYnka6x+pK1I4HIy4CYOk86FgG/4r+q33IqF9S/W4/X +2UpMLhqis11/z3HiFxJc0IIhiWRK2ybZ084RkvNEILQAfAcFkCE9s4KBnGCX+6Tx +b5nDDKjBhcCtHTAvixz5Pnj7Rc/HbJCNlyfI4AXbbmXJw25TLlRrY2E+p9K7M70X +aNdfPppRgiz8zn7DtSXTqi1gEfPMmX6ghbdU68Js6UU9XKyUHjOou5rsOGjtC+7k +npPUDOgDx89EHsUqu/8xzrUYwOI5yWw82SR0ZcDnZbDVPXJk8+rliEF5YPtqLw+e +WLqzcvo2H5nw2KYpoaHhn+eM2hE6dX07kyWKcQIDAQABAoIBAQChriKuza0MfBri +9x3UCRN/is/wDZVe1P+2KL8F9ZvPxytNVeP4qM7c38WzF8MQ6sRR8z0WiqCZOjj4 +f3QX7iG2MlAvUkUqAFk778ZIuUov5sE/bU8RLOrfJKz1vqOLa2w8/xHH5LwS1/jn +m6t9zZPCSwpMiMSUSZci1xQlS6b6POZMjeqLPqv9cP8PJNv9UNrHFcQnQi1iwKJH +MJ7CQI3R8FSeGad3P7tB9YDaBm7hHmd/TevuFkymcKNT44XBSgddPDfgKui6sHTY +QQWgWI9VGVO350ZBLRLkrk8wboY4vc15qbBzYFG66WiR/tNdLt3rDYxpcXaDvcQy +e47mYNVxAoGBAMFsUmPDssqzmOkmZxHDM+VmgHYPXjDqQdE299FtuClobUW4iU4g +By7o84aCIBQz2sp9f1KM+10lr+Bqw3s7QBbR5M67PA8Zm45DL9t70NR/NZHGzFRD +BR/NMbwzCqNtY2UGDhYQLGhW8heAwsYwir8ZqmOfKTd9aY1pu/S8m9AlAoGBAP6I +483EIN8R5y+beGcGynYeIrH5Gc+W2FxWIW9jh/G7vRbhMlW4z0GxV3uEAYmOlBH2 +AqUkV6+uzU0P4B/m3vCYqLycBVDwifJazDj9nskVL5kGMxia62iwDMXs5nqNS4WJ +ZM5Gl2xIiwmgWnYnujM3eKF2wbm439wj4na80SldAoGANdIqatA9o+GtntKsw2iJ +vD91Z2SHVR0aC1k8Q+4/3GXOYiQjMLYAybDQcpEq0/RJ4SZik1nfZ9/gvJV4p4Wp +I7Br9opq/9ikTEWtv2kIhtiO02151ciAWIUEXdXmE+uQSMASk1kUwkPPQXL2v6cq +NFqz6tyS33nqMQtG3abNxHECgYA4AEA2nmcpDRRTSh50dG8JC9pQU+EU5jhWIHEc +w8Y+LjMNHKDpcU7QQkdgGowICsGTLhAo61ULhycORGboPfBg+QVu8djNlQ6Urttt +0ocj8LBXN6D4UeVnVAyLY3LWFc4+5Bq0s51PKqrEhG5Cvrzd1d+JjspSpVVDZvXF +cAeI1QKBgC/cMN3+2Sc+2biu46DnkdYpdF/N0VGMOgzz+unSVD4RA2mEJ9UdwGga +feshtrtcroHtEmc+WDYgTTnAq1MbsVFQYIwZ5fL/GJ1R8ccaWiPuX2HrKALKG4Y3 +CMFpDUWhRgtaBsmuOpUq3FeS5cyPNMHk6axL1KyFoJk9AgfhqhTp +-----END RSA PRIVATE KEY----- diff --git a/pkg/acquisition/modules/http/testdata/server.crt b/pkg/acquisition/modules/http/testdata/server.crt new file mode 100644 index 00000000000..7a02c606c9d --- /dev/null +++ b/pkg/acquisition/modules/http/testdata/server.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID5jCCAs6gAwIBAgIUU3F6URi0oTe9ontkf7JqXOo89QYwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCRlIxFjAUBgNVBAgTDUlsZS1kZS1GcmFuY2UxDjAMBgNV +BAcTBVBhcmlzMREwDwYDVQQKEwhDcm93ZHNlYzERMA8GA1UECxMIQ3Jvd2RzZWMx +EjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0yNDEwMjMxMDAzMDBaFw0yNTEwMjMxMDAz +MDBaMG8xCzAJBgNVBAYTAkZSMRYwFAYDVQQIEw1JbGUtZGUtRnJhbmNlMQ4wDAYD +VQQHEwVQYXJpczERMA8GA1UEChMIQ3Jvd2RzZWMxETAPBgNVBAsTCENyb3dkc2Vj +MRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC/lnUubjBGe5x0LgIE5GeG52LRzj99iLWuvey4qbSwFZ07ECgv+JttVwDm +AjEeakj2ZR46WHvHAR9eBNkRCORyWX0iKVIzm09PXYi80KtwGLaA8YMEio9/08Cc ++LS0TuP0yiOcw+btrhmvvauDzcQhA6u55q8anCZiF2BlHfX9Sh6QKewA3NhOkzbU +VTxqrOqfcRsGNub7dheqfP5bfrPkF6Y6l/0Fhyx0NMsu1zaQ0hCls2hkTf0Y3XGt +IQNWoN22seexR3qRmPf0j3jBa0qOmGgd6kAd+YpsjDblgCNUIJZiVj51fVb0sGRx +ShkfKGU6t0eznTWPCqswujO/sn+pAgMBAAGjejB4MA4GA1UdDwEB/wQEAwIFoDAd +BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV +HQ4EFgQUOiIF+7Wzx1J8Ki3DiBfx+E6zlSUwGgYDVR0RBBMwEYIJbG9jYWxob3N0 +hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQA0dzlhBr/0wXPyj/iWxMOXxZ1FNJ9f +lxBMhLAgX0WrT2ys+284J7Hcn0lJeqelluYpmeKn9vmCAEj3MmUmHzZyf//lhuUJ +0DlYWIHUsGaJHJ7A+1hQqrcXHhkcRy5WGIM9VoddKbBbg2b6qzTSvxn8EnuD7H4h +28wLyGLCzsSXoVcAB8u+svYt29TPuy6xmMAokyIShV8FsE77fjVTgtCuxmx1PKv3 +zd6+uEae7bbZ+GJH1zKF0vokejQvmByt+YuIXlNbMseaMUeDdpy+6qlRvbbN1dyp +rkQXfWvidMfSue5nH/akAn83v/CdKxG6tfW83d9Rud3naabUkywALDng +-----END CERTIFICATE----- diff --git a/pkg/acquisition/modules/http/testdata/server.key b/pkg/acquisition/modules/http/testdata/server.key new file mode 100644 index 00000000000..4d0ee53b4c2 --- /dev/null +++ b/pkg/acquisition/modules/http/testdata/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAv5Z1Lm4wRnucdC4CBORnhudi0c4/fYi1rr3suKm0sBWdOxAo +L/ibbVcA5gIxHmpI9mUeOlh7xwEfXgTZEQjkcll9IilSM5tPT12IvNCrcBi2gPGD +BIqPf9PAnPi0tE7j9MojnMPm7a4Zr72rg83EIQOrueavGpwmYhdgZR31/UoekCns +ANzYTpM21FU8aqzqn3EbBjbm+3YXqnz+W36z5BemOpf9BYcsdDTLLtc2kNIQpbNo +ZE39GN1xrSEDVqDdtrHnsUd6kZj39I94wWtKjphoHepAHfmKbIw25YAjVCCWYlY+ +dX1W9LBkcUoZHyhlOrdHs501jwqrMLozv7J/qQIDAQABAoIBAF1Vd/rJlV0Q5RQ4 +QaWOe9zdpmedeZK3YgMh5UvE6RCLRxC5+0n7bASlSPvEf5dYofjfJA26g3pcUqKj +6/d/hIMsk2hsBu67L7TzVSTe51XxxB8nCPPSaLwWNZSDGM1qTWU4gIbjbQHHOh5C +YWcRfAW1WxhyiEWHYq+QwdYg9XCRrSg1UzvVvW1Yt2wDGcSZP5whbXipfw3BITDs +XU7ODYNkU1sjIzQZwzVGxOf9qKdhZFZ26Vhoz8OJNMLyJxY7EspuwR7HbDGt11Pb +CxOt/BV44LwdVYeqh57oIKtckQW33W/6EeaWr7GfMzyH5WSrsOJoK5IJVrZaPTcS +QiMYLA0CgYEA9vMVsGshBl3TeRGaU3XLHqooXD4kszbdnjfPrwGlfCO/iybhDqo5 +WFypM/bYcIWzbTez/ihufHEHPSCUbFEcN4B+oczGcuxTcZjFyvJYvq2ycxPUiDIi +JnVUcVxgh1Yn39+CsQ/b6meP7MumTD2P3I87CeQGlWTO5Ys9mdw0BjcCgYEAxpv1 +64l5UoFJGr4yElNKDIKnhEFbJZsLGKiiuVXcS1QVHW5Az5ar9fPxuepyHpz416l3 +ppncuhJiUIP+jbu5e0s0LsN46mLS3wkHLgYJj06CNT3uOSLSg1iFl7DusdbyiaA7 +wEJ/aotS1NZ4XaeryAWHwYJ6Kag3nz6NV3ZYuR8CgYEAxAFCuMj+6F+2RsTa+d1n +v8oMyNImLPyiQD9KHzyuTW7OTDMqtIoVg/Xf8re9KOpl9I0e1t7eevT3auQeCi8C +t2bMm7290V+UB3jbnO5n08hn+ADIUuV/x4ie4m8QyrpuYbm0sLbGtTFHwgoNzzuZ +oNUqZfpP42mk8fpnhWSLAlcCgYEAgpY7XRI4HkJ5ocbav2faMV2a7X/XgWNvKViA +HeJRhYoUlBRRMuz7xi0OjFKVlIFbsNlxna5fDk1WLWCMd/6tl168Qd8u2tX9lr6l +5OH9WSeiv4Un5JN73PbQaAvi9jXBpTIg92oBwzk2TlFyNQoxDcRtHZQ/5LIBWIhV +gOOEtLsCgYEA1wbGc4XlH+/nXVsvx7gmfK8pZG8XA4/ToeIEURwPYrxtQZLB4iZs +aqWGgIwiB4F4UkuKZIjMrgInU9y0fG6EL96Qty7Yjh7dGy1vJTZl6C+QU6o4sEwl +r5Id5BNLEaqISWQ0LvzfwdfABYlvFfBdaGbzUzLEitD79eyhxuNEOBw= +-----END RSA PRIVATE KEY----- From 4fc9e2dd42cb46e7917f91bc81e1b0d02a104d08 Mon Sep 17 00:00:00 2001 From: he2ss Date: Wed, 23 Oct 2024 15:18:39 +0200 Subject: [PATCH 02/30] remove unwanted comments --- pkg/acquisition/modules/http/http.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index c7347feb772..2477d18d7ab 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -9,8 +9,6 @@ import ( "os" "time" - // import gin for http server - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -133,7 +131,6 @@ func (hc *HttpConfiguration) Validate() error { return fmt.Errorf("chunk_size must be positive") } - // validate custom status code if hc.CustomStatusCode != nil { statusText := http.StatusText(*hc.CustomStatusCode) if statusText == "" { From 8315cdd37595f1a7b4f6a4b34015751e0fc38123 Mon Sep 17 00:00:00 2001 From: he2ss Date: Wed, 23 Oct 2024 15:19:21 +0200 Subject: [PATCH 03/30] remove unused code --- pkg/acquisition/modules/http/http.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 2477d18d7ab..f2f4d1a7dc6 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -32,8 +32,6 @@ var linesRead = prometheus.NewCounterVec( }, []string{"path"}) -const () - type HttpConfiguration struct { //IPFilter []string `yaml:"ip_filter"` Port int `yaml:"port"` From 066bda9b08f065c68fdbf4387def8d999e2f05e6 Mon Sep 17 00:00:00 2001 From: he2ss Date: Wed, 23 Oct 2024 16:59:55 +0200 Subject: [PATCH 04/30] add server close on dying tomb + acquis mode --- pkg/acquisition/acquisition.go | 6 ++-- pkg/acquisition/modules/http/http.go | 43 ++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index b947b6f15ee..6bd12a8cb2a 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -21,6 +21,7 @@ import ( cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" + httpacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/http" journalctlacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/journalctl" kafkaacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kafka" kinesisacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kinesis" @@ -36,7 +37,7 @@ import ( type DataSourceUnavailableError struct { Name string - Err error + Err error } func (e *DataSourceUnavailableError) Error() string { @@ -47,7 +48,6 @@ func (e *DataSourceUnavailableError) Unwrap() error { return e.Err } - // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module @@ -75,6 +75,7 @@ var AcquisitionSources = map[string]func() DataSource{ "kafka": func() DataSource { return &kafkaacquisition.KafkaSource{} }, "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, "s3": func() DataSource { return &s3acquisition.S3Source{} }, + "http": func() DataSource { return &httpacquisition.HTTPSource{} }, } var transformRuntimes = map[string]*vm.Program{} @@ -343,6 +344,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb return nil }) } + log.Tracef("Mode is %s", subsrc.GetMode()) if subsrc.GetMode() == configuration.TAIL_MODE { err = subsrc.StreamingAcquisition(outChan, AcquisTomb) } else { diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index f2f4d1a7dc6..bd4b269edc6 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -76,6 +76,11 @@ func (h *HTTPSource) UnmarshalConfig(yamlConfig []byte) error { if err != nil { return fmt.Errorf("cannot parse %s datasource configuration: %w", dataSourceName, err) } + + if h.Config.Mode == "" { + h.Config.Mode = configuration.TAIL_MODE + } + return nil } @@ -319,21 +324,35 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { h.Server.TLSConfig = tlsConfig } - if h.Config.TLS != nil { - h.logger.Infof("start https server on port %d", h.Config.Port) - err := h.Server.ListenAndServeTLS(h.Config.TLS.ServerCert, h.Config.TLS.ServerKey) - if err != nil && err != http.ErrServerClosed { - return fmt.Errorf("https server failed: %w", err) + t.Go(func() error { + defer trace.CatchPanic("crowdsec/acquis/http/server") + if h.Config.TLS != nil { + h.logger.Infof("start https server on port %d", h.Config.Port) + err := h.Server.ListenAndServeTLS(h.Config.TLS.ServerCert, h.Config.TLS.ServerKey) + if err != nil && err != http.ErrServerClosed { + return fmt.Errorf("https server failed: %w", err) + } + } else { + h.logger.Infof("start http server on port %d", h.Config.Port) + err := h.Server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + return fmt.Errorf("http server failed: %w", err) + } } - } else { - h.logger.Infof("start http server on port %d", h.Config.Port) - err := h.Server.ListenAndServe() - if err != nil && err != http.ErrServerClosed { - return fmt.Errorf("http server failed: %w", err) + return nil + }) + + //nolint //fp + for { + select { + case <-t.Dying(): + h.logger.Infof("%s datasource stopping", dataSourceName) + if err := h.Server.Close(); err != nil { + return fmt.Errorf("while closing %s server: %w", dataSourceName, err) + } + return nil } } - - return nil } func (h *HTTPSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { From 5577e10d87f1d8f607dc7f3ce65c65ae1b6c15ae Mon Sep 17 00:00:00 2001 From: he2ss Date: Wed, 23 Oct 2024 17:06:30 +0200 Subject: [PATCH 05/30] remove unwanted trace --- pkg/acquisition/acquisition.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 6bd12a8cb2a..11d5320e5f1 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -344,7 +344,6 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb return nil }) } - log.Tracef("Mode is %s", subsrc.GetMode()) if subsrc.GetMode() == configuration.TAIL_MODE { err = subsrc.StreamingAcquisition(outChan, AcquisTomb) } else { From 1784d54e7a81c7d08d1e308b92d3ea56d6ec1451 Mon Sep 17 00:00:00 2001 From: he2ss Date: Wed, 23 Oct 2024 18:07:54 +0200 Subject: [PATCH 06/30] fix branch --- pkg/acquisition/modules/http/http.go | 24 ++++++++++++++----- pkg/acquisition/modules/http/http_test.go | 10 ++++---- pkg/cwversion/component/component.go | 29 ++++++++++++----------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index bd4b269edc6..87a73b5be20 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -1,6 +1,7 @@ package httpacquisition import ( + "context" "crypto/tls" "crypto/x509" "fmt" @@ -15,7 +16,7 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -61,9 +62,10 @@ type TLSConfig struct { } type HTTPSource struct { - Config HttpConfiguration - logger *log.Entry - Server *http.Server + metricsLevel int + Config HttpConfiguration + logger *log.Entry + Server *http.Server } func (h *HTTPSource) GetUuid() string { @@ -144,8 +146,9 @@ func (hc *HttpConfiguration) Validate() error { return nil } -func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry) error { +func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { h.logger = logger + h.metricsLevel = MetricsLevel err := h.UnmarshalConfig(yamlConfig) if err != nil { return err @@ -260,12 +263,21 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * Module: h.GetName(), } + if h.metricsLevel == configuration.METRICS_AGGREGATE { + line.Src = hc.Path + } + evt := types.Event{ Line: line, Process: true, Type: types.LOG, ExpectMode: types.LIVE, } + + if h.metricsLevel != configuration.METRICS_NONE { + linesRead.With(prometheus.Labels{"path": hc.Path}).Inc() + } + out <- evt return nil }) @@ -355,7 +367,7 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { } } -func (h *HTTPSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { +func (h *HTTPSource) StreamingAcquisition(ctx context.Context, out chan types.Event, t *tomb.Tomb) error { h.logger.Debugf("start http server on port %d", h.Config.Port) t.Go(func() error { diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 3752496aefc..84643834d70 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -1,6 +1,7 @@ package httpacquisition import ( + "context" "crypto/tls" "crypto/x509" "fmt" @@ -12,7 +13,7 @@ import ( "time" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/go-cs-lib/pkg/cstest" + "github.com/crowdsecurity/go-cs-lib/cstest" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" ) @@ -184,7 +185,7 @@ custom_status_code: 999`, for _, test := range tests { h := HTTPSource{} - err := h.Configure([]byte(test.config), subLogger) + err := h.Configure([]byte(test.config), subLogger, 0) cstest.AssertErrorContains(t, err, test.expectedErr) } } @@ -235,16 +236,17 @@ func TestGetName(t *testing.T) { } func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte) (chan types.Event, *tomb.Tomb) { + ctx := context.Background() subLogger := log.WithFields(log.Fields{ "type": "http", }) - err := h.Configure(config, subLogger) + err := h.Configure(config, subLogger, 0) if err != nil { t.Fatalf("unable to configure http source: %s", err) } tomb := tomb.Tomb{} out := make(chan types.Event) - err = h.StreamingAcquisition(out, &tomb) + err = h.StreamingAcquisition(ctx, out, &tomb) if err != nil { t.Fatalf("unable to start streaming acquisition: %s", err) } diff --git a/pkg/cwversion/component/component.go b/pkg/cwversion/component/component.go index 4036b63cf00..7ed596525e0 100644 --- a/pkg/cwversion/component/component.go +++ b/pkg/cwversion/component/component.go @@ -7,20 +7,21 @@ package component // Built is a map of all the known components, and whether they are built-in or not. // This is populated as soon as possible by the respective init() functions -var Built = map[string]bool { - "datasource_appsec": false, - "datasource_cloudwatch": false, - "datasource_docker": false, - "datasource_file": false, - "datasource_journalctl": false, - "datasource_k8s-audit": false, - "datasource_kafka": false, - "datasource_kinesis": false, - "datasource_loki": false, - "datasource_s3": false, - "datasource_syslog": false, - "datasource_wineventlog":false, - "cscli_setup": false, +var Built = map[string]bool{ + "datasource_appsec": false, + "datasource_cloudwatch": false, + "datasource_docker": false, + "datasource_file": false, + "datasource_journalctl": false, + "datasource_k8s-audit": false, + "datasource_kafka": false, + "datasource_kinesis": false, + "datasource_loki": false, + "datasource_s3": false, + "datasource_syslog": false, + "datasource_wineventlog": false, + "datasource_http": false, + "cscli_setup": false, } func Register(name string) { From 329a957d0e22bfe662e8235c8d8abc2fe42f6f27 Mon Sep 17 00:00:00 2001 From: he2ss Date: Thu, 24 Oct 2024 09:52:55 +0200 Subject: [PATCH 07/30] fix lint --- pkg/acquisition/modules/http/http.go | 6 +++--- pkg/acquisition/modules/http/http_test.go | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 87a73b5be20..f983b086ed0 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -14,7 +14,7 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/crowdsecurity/go-cs-lib/trace" @@ -173,7 +173,7 @@ func (h *HTTPSource) GetName() string { return dataSourceName } -func (k *HTTPSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { +func (h *HTTPSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { return fmt.Errorf("%s datasource does not support one-shot acquisition", dataSourceName) } @@ -185,7 +185,7 @@ func (h *HTTPSource) GetMetrics() []prometheus.Collector { return []prometheus.Collector{linesRead} } -func (k *HTTPSource) GetAggregMetrics() []prometheus.Collector { +func (h *HTTPSource) GetAggregMetrics() []prometheus.Collector { return []prometheus.Collector{linesRead} } diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 84643834d70..1440bbde84d 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -366,7 +366,7 @@ headers: if err != nil { t.Fatalf("unable to create http request: %s", err) } - req.Header.Add("key", "wrong") + req.Header.Add("Key", "wrong") resp, err := client.Do(req) if err != nil { t.Fatalf("unable to post http request: %s", err) @@ -398,7 +398,7 @@ max_body_size: 5`)) if err != nil { t.Fatalf("unable to create http request: %s", err) } - req.Header.Add("key", "test") + req.Header.Add("Key", "test") resp, err := client.Do(req) if err != nil { t.Fatalf("unable to post http request: %s", err) @@ -430,7 +430,7 @@ headers: if err != nil { t.Fatalf("unable to create http request: %s", err) } - req.Header.Add("key", "test") + req.Header.Add("Key", "test") resp, err := client.Do(req) if err != nil { t.Fatalf("unable to post http request: %s", err) @@ -466,7 +466,7 @@ custom_headers: if err != nil { t.Fatalf("unable to create http request: %s", err) } - req.Header.Add("key", "test") + req.Header.Add("Key", "test") resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("unable to post http request: %s", err) @@ -476,7 +476,7 @@ custom_headers: t.Fatalf("expected status code %d, got %d", http.StatusCreated, resp.StatusCode) } - if resp.Header.Get("success") != "true" { + if resp.Header.Get("Success") != "true" { t.Fatalf("expected header 'success' to be 'true', got '%s'", resp.Header.Get("success")) } @@ -539,11 +539,11 @@ timeout: 1s`)) body: []byte(`{"test": "delayed_payload"}`), } - req, err := http.NewRequest("POST", fmt.Sprintf("%s/test", testHTTPServerAddr), slow) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), slow) if err != nil { t.Fatalf("Error creating request: %v", err) } - req.Header.Add("key", "test") + req.Header.Add("Key", "test") req.Header.Set("Content-Type", "application/json") client := &http.Client{} @@ -626,7 +626,7 @@ tls: if err != nil { t.Fatalf("unable to create http request: %s", err) } - req.Header.Add("key", "test") + req.Header.Add("Key", "test") resp, err := client.Do(req) if err != nil { t.Fatalf("unable to post http request: %s", err) From fc29db90fdc0c218d335667fa3477edbe2e25126 Mon Sep 17 00:00:00 2001 From: he2ss Date: Thu, 24 Oct 2024 12:12:34 +0200 Subject: [PATCH 08/30] make transform unmarshal works + fix lint --- .../modules/cloudwatch/cloudwatch.go | 1 + pkg/acquisition/modules/docker/docker.go | 3 +- pkg/acquisition/modules/file/file.go | 4 +- pkg/acquisition/modules/http/http.go | 42 ++++++++++--------- .../modules/journalctl/journalctl.go | 1 + pkg/acquisition/modules/kafka/kafka.go | 2 +- pkg/acquisition/modules/kinesis/kinesis.go | 1 + .../modules/kubernetesaudit/k8s_audit.go | 9 ++-- pkg/acquisition/modules/loki/loki.go | 9 ++-- pkg/acquisition/modules/s3/s3.go | 1 + pkg/acquisition/modules/syslog/syslog.go | 4 +- .../wineventlog/wineventlog_windows.go | 6 +-- 12 files changed, 46 insertions(+), 37 deletions(-) diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index e4b6c95d77f..52309ac647c 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -732,6 +732,7 @@ func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig) evt.Process = true evt.Type = types.LOG evt.ExpectMode = cfg.ExpectMode + evt.Unmarshaled = make(map[string]interface{}) cfg.logger.Debugf("returned event labels : %+v", evt.Line.Labels) return evt, nil } diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index 874b1556fd5..f54e7d9f579 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -334,7 +334,7 @@ func (d *DockerSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) er if d.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"source": containerConfig.Name}).Inc() } - evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} + evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} out <- evt d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw) } @@ -580,6 +580,7 @@ func (d *DockerSource) TailDocker(container *ContainerConfig, outChan chan types l.Process = true l.Module = d.GetName() var evt types.Event + evt.Unmarshaled = make(map[string]interface{}) if !d.Config.UseTimeMachine { evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} } else { diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 2d2df3ff4d4..e027f5d8025 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -625,7 +625,7 @@ func (f *FileSource) tailFile(out chan types.Event, t *tomb.Tomb, tail *tail.Tai if f.config.UseTimeMachine { expectMode = types.TIMEMACHINE } - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: expectMode} + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: expectMode, Unmarshaled: make(map[string]interface{})} } } } @@ -684,7 +684,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom linesRead.With(prometheus.Labels{"source": filename}).Inc() // we're reading logs at once, it must be time-machine buckets - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} } } diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index f983b086ed0..91d788371db 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net/http" @@ -88,58 +89,58 @@ func (h *HTTPSource) UnmarshalConfig(yamlConfig []byte) error { func (hc *HttpConfiguration) Validate() error { if hc.Port == 0 { - return fmt.Errorf("port is required") + return errors.New("port is required") } if hc.Path == "" { - return fmt.Errorf("path is required") + return errors.New("path is required") } switch hc.AuthType { case "basic_auth": if hc.BasicAuth == nil { - return fmt.Errorf("basic_auth is required") + return errors.New("basic_auth is required") } if hc.BasicAuth.Username == "" { - return fmt.Errorf("username is required") + return errors.New("username is required") } if hc.BasicAuth.Password == "" { - return fmt.Errorf("password is required") + return errors.New("password is required") } case "headers": if hc.Headers == nil { - return fmt.Errorf("headers is required") + return errors.New("headers is required") } case "mtls": if hc.TLS == nil || hc.TLS != nil && hc.TLS.CaCert == "" { - return fmt.Errorf("ca_cert is required") + return errors.New("ca_cert is required") } default: if hc.TLS == nil { - return fmt.Errorf("at least one of tls or auth_type is required") + return errors.New("at least one of tls or auth_type is required") } } if hc.TLS != nil { if hc.TLS.ServerCert == "" { - return fmt.Errorf("server_cert is required") + return errors.New("server_cert is required") } if hc.TLS.ServerKey == "" { - return fmt.Errorf("server_key is required") + return errors.New("server_key is required") } } if hc.MaxBodySize != nil && *hc.MaxBodySize <= 0 { - return fmt.Errorf("max_body_size must be positive") + return errors.New("max_body_size must be positive") } if hc.ChunkSize != nil && *hc.ChunkSize <= 0 { - return fmt.Errorf("chunk_size must be positive") + return errors.New("chunk_size must be positive") } if hc.CustomStatusCode != nil { statusText := http.StatusText(*hc.CustomStatusCode) if statusText == "" { - return fmt.Errorf("invalid HTTP status code") + return errors.New("invalid HTTP status code") } } @@ -224,16 +225,16 @@ func authorizeRequest(r *http.Request, hc *HttpConfiguration) error { if hc.AuthType == "basic_auth" { username, password, ok := r.BasicAuth() if !ok { - return fmt.Errorf("missing basic auth") + return errors.New("missing basic auth") } if username != hc.BasicAuth.Username || password != hc.BasicAuth.Password { - return fmt.Errorf("invalid basic auth") + return errors.New("invalid basic auth") } } if hc.AuthType == "headers" { for key, value := range *hc.Headers { if r.Header.Get(key) != value { - return fmt.Errorf("invalid headers") + return errors.New("invalid headers") } } } @@ -268,10 +269,11 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * } evt := types.Event{ - Line: line, - Process: true, - Type: types.LOG, - ExpectMode: types.LIVE, + Line: line, + Process: true, + Type: types.LOG, + ExpectMode: types.LIVE, + Unmarshaled: make(map[string]interface{}), } if h.metricsLevel != configuration.METRICS_NONE { diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index b9cda54a472..2ee811475a5 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -137,6 +137,7 @@ func (j *JournalCtlSource) runJournalCtl(out chan types.Event, t *tomb.Tomb) err linesRead.With(prometheus.Labels{"source": j.src}).Inc() } var evt types.Event + evt.Unmarshaled = make(map[string]interface{}) if !j.config.UseTimeMachine { evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} } else { diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 9fd5fc2a035..7d66ca9ae78 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -174,7 +174,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error { linesRead.With(prometheus.Labels{"topic": k.Config.Topic}).Inc() } var evt types.Event - + evt.Unmarshaled = make(map[string]interface{}) if !k.Config.UseTimeMachine { evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} } else { diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index ca3a847dbfb..811d3e932d0 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -323,6 +323,7 @@ func (k *KinesisSource) ParseAndPushRecords(records []*kinesis.Record, out chan l.Src = k.Config.StreamName } var evt types.Event + evt.Unmarshaled = make(map[string]interface{}) if !k.Config.UseTimeMachine { evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} } else { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index f979b044dcc..d6fd668605d 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -208,10 +208,11 @@ func (ka *KubernetesAuditSource) webhookHandler(w http.ResponseWriter, r *http.R Module: ka.GetName(), } ka.outChan <- types.Event{ - Line: l, - Process: true, - Type: types.LOG, - ExpectMode: types.LIVE, + Line: l, + Process: true, + Type: types.LOG, + ExpectMode: types.LIVE, + Unmarshaled: make(map[string]interface{}), } } } diff --git a/pkg/acquisition/modules/loki/loki.go b/pkg/acquisition/modules/loki/loki.go index f867feeb84b..9ab88044cff 100644 --- a/pkg/acquisition/modules/loki/loki.go +++ b/pkg/acquisition/modules/loki/loki.go @@ -312,10 +312,11 @@ func (l *LokiSource) readOneEntry(entry lokiclient.Entry, labels map[string]stri expectMode = types.TIMEMACHINE } out <- types.Event{ - Line: ll, - Process: true, - Type: types.LOG, - ExpectMode: expectMode, + Line: ll, + Process: true, + Type: types.LOG, + ExpectMode: expectMode, + Unmarshaled: make(map[string]interface{}), } } diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index ed1964edebf..407f75ed9ea 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -444,6 +444,7 @@ func (s *S3Source) readFile(bucket string, key string) error { l.Src = bucket } var evt types.Event + evt.Unmarshaled = make(map[string]interface{}) if !s.Config.UseTimeMachine { evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} } else { diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 5315096fb9b..108cec60e25 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -236,9 +236,9 @@ func (s *SyslogSource) handleSyslogMsg(out chan types.Event, t *tomb.Tomb, c cha l.Src = syslogLine.Client l.Process = true if !s.config.UseTimeMachine { - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE, Unmarshaled: make(map[string]interface{})} } else { - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} } } } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index ca40363155b..e13efda46c7 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -206,9 +206,9 @@ func (w *WinEventLogSource) getEvents(out chan types.Event, t *tomb.Tomb) error l.Src = w.name l.Process = true if !w.config.UseTimeMachine { - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE, Unmarshaled: make(map[string]interface{})} } else { - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} } } } @@ -432,7 +432,7 @@ OUTER_LOOP: l.Time = time.Now() l.Src = w.name l.Process = true - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} + out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} } } } From 9bc0f747bbd2396845f880f73c785e1506308d65 Mon Sep 17 00:00:00 2001 From: he2ss Date: Thu, 24 Oct 2024 15:33:36 +0200 Subject: [PATCH 09/30] handle gzip encoding --- pkg/acquisition/modules/http/http.go | 17 +++++++- pkg/acquisition/modules/http/http_test.go | 53 ++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 91d788371db..68dd3c88f30 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -1,6 +1,7 @@ package httpacquisition import ( + "compress/gzip" "context" "crypto/tls" "crypto/x509" @@ -241,12 +242,26 @@ func authorizeRequest(r *http.Request, hc *HttpConfiguration) error { return nil } +func ReadBody(r *http.Request) ([]byte, error) { + if r.Header.Get("Content-Encoding") == "gzip" { + body, err := gzip.NewReader(r.Body) + if err != nil { + return nil, fmt.Errorf("failed to create gzip reader: %w", err) + } + defer body.Close() + return io.ReadAll(body) + } + + return io.ReadAll(r.Body) +} + func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc *HttpConfiguration, out chan types.Event, t *tomb.Tomb) error { if hc.MaxBodySize != nil && r.ContentLength > *hc.MaxBodySize { w.WriteHeader(http.StatusRequestEntityTooLarge) return fmt.Errorf("body size exceeds max body size: %d > %d", r.ContentLength, *hc.MaxBodySize) } - body, err := io.ReadAll(r.Body) + + body, err := ReadBody(r) defer r.Body.Close() if err != nil { w.WriteHeader(http.StatusBadRequest) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 1440bbde84d..ab43f0cf17e 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -1,6 +1,7 @@ package httpacquisition import ( + "compress/gzip" "context" "crypto/tls" "crypto/x509" @@ -205,7 +206,7 @@ source: http port: 8080 path: 15 auth_type: headers`)) - cstest.AssertErrorMessage(t, err, "cannot parse http datasource configuration: yaml: line 5: found a tab character that violates indentation") + cstest.AssertErrorMessage(t, err, "cannot parse http datasource configuration: yaml: line 4: found a tab character that violates indentation") } func TestConfigureByDSN(t *testing.T) { @@ -702,3 +703,53 @@ tls: tomb.Kill(nil) tomb.Wait() } + +func TestStreamingAcquisitionGzipData(t *testing.T) { + h := &HTTPSource{} + out, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test`)) + + time.Sleep(1 * time.Second) + + rawEvt := `{"test": "test"}` + + // send gzipped compressed data + client := &http.Client{} + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.Header.Add("Key", "test") + req.Header.Add("Content-Encoding", "gzip") + req.Header.Add("Content-Type", "application/json") + + var b strings.Builder + gz := gzip.NewWriter(&b) + if _, err := gz.Write([]byte(rawEvt)); err != nil { + t.Fatalf("unable to write gzipped data: %s", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("unable to close gzip writer: %s", err) + } + req.Body = io.NopCloser(strings.NewReader(b.String())) + req.ContentLength = int64(b.Len()) + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + assertEvents(out, t, rawEvt) + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} From 47131848cd9d52fd84f26556f95f08e247d754dd Mon Sep 17 00:00:00 2001 From: he2ss Date: Thu, 24 Oct 2024 15:54:05 +0200 Subject: [PATCH 10/30] fix lint --- pkg/acquisition/modules/http/http_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index ab43f0cf17e..b7f7999818f 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -478,7 +478,7 @@ custom_headers: } if resp.Header.Get("Success") != "true" { - t.Fatalf("expected header 'success' to be 'true', got '%s'", resp.Header.Get("success")) + t.Fatalf("expected header 'success' to be 'true', got '%s'", resp.Header.Get("Success")) } assertEvents(out, t, rawEvt) From 0e6031c054743c849aa85fbb217141c16cef7e16 Mon Sep 17 00:00:00 2001 From: he2ss Date: Thu, 24 Oct 2024 16:08:26 +0200 Subject: [PATCH 11/30] assert line.src in tests --- pkg/acquisition/modules/http/http.go | 8 +++++++- pkg/acquisition/modules/http/http_test.go | 18 ++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 68dd3c88f30..d53e713c174 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "os" "time" @@ -269,10 +270,15 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * } h.logger.Tracef("body received: %+v", string(body)) + srcHost, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return err + } + t.Go(func() error { line := types.Line{ Raw: string(body), - Src: r.RemoteAddr, + Src: srcHost, Time: time.Now().UTC(), Labels: hc.Labels, Process: true, diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index b7f7999818f..0bf3cff8058 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -440,7 +440,7 @@ headers: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertEvents(out, t, rawEvt) + assertFirstEvent(out, t, rawEvt) h.Server.Close() tomb.Kill(nil) @@ -481,7 +481,7 @@ custom_headers: t.Fatalf("expected header 'success' to be 'true', got '%s'", resp.Header.Get("Success")) } - assertEvents(out, t, rawEvt) + assertFirstEvent(out, t, rawEvt) h.Server.Close() tomb.Kill(nil) @@ -504,7 +504,7 @@ func (sr *slowReader) Read(p []byte) (int, error) { return n, nil } -func assertEvents(out chan types.Event, t *testing.T, expected string) { +func assertFirstEvent(out chan types.Event, t *testing.T, expected string) { readLines := []types.Event{} select { @@ -520,6 +520,12 @@ func assertEvents(out chan types.Event, t *testing.T, expected string) { if readLines[0].Line.Raw != expected { t.Fatalf(`expected %s, got '%+v'`, expected, readLines[0].Line) } + if readLines[0].Line.Src != "127.0.0.1" { + t.Fatalf("expected '127.0.0.1', got '%s'", readLines[0].Line.Src) + } + if readLines[0].Line.Module != "http" { + t.Fatalf("expected 'http', got '%s'", readLines[0].Line.Module) + } } func TestStreamingAcquisitionTimeout(t *testing.T) { @@ -636,7 +642,7 @@ tls: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertEvents(out, t, rawEvt) + assertFirstEvent(out, t, rawEvt) h.Server.Close() tomb.Kill(nil) @@ -697,7 +703,7 @@ tls: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertEvents(out, t, rawEvt) + assertFirstEvent(out, t, rawEvt) h.Server.Close() tomb.Kill(nil) @@ -747,7 +753,7 @@ headers: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertEvents(out, t, rawEvt) + assertFirstEvent(out, t, rawEvt) h.Server.Close() tomb.Kill(nil) From e9bf32e8c0f0756aab30f989a5252fa55fc36472 Mon Sep 17 00:00:00 2001 From: he2ss Date: Thu, 24 Oct 2024 20:47:46 +0200 Subject: [PATCH 12/30] validate path + better naming --- pkg/acquisition/modules/http/http.go | 9 ++++++--- pkg/acquisition/modules/http/http_test.go | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index d53e713c174..15f1958aea8 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -96,6 +96,9 @@ func (hc *HttpConfiguration) Validate() error { if hc.Path == "" { return errors.New("path is required") } + if hc.Path[0] != '/' { + return errors.New("path must start with /") + } switch hc.AuthType { case "basic_auth": @@ -245,12 +248,12 @@ func authorizeRequest(r *http.Request, hc *HttpConfiguration) error { func ReadBody(r *http.Request) ([]byte, error) { if r.Header.Get("Content-Encoding") == "gzip" { - body, err := gzip.NewReader(r.Body) + gReader, err := gzip.NewReader(r.Body) if err != nil { return nil, fmt.Errorf("failed to create gzip reader: %w", err) } - defer body.Close() - return io.ReadAll(body) + defer gReader.Close() + return io.ReadAll(gReader) } return io.ReadAll(r.Body) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 0bf3cff8058..29221b5dc49 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -50,6 +50,13 @@ port: 8080`, config: ` source: http port: 8080 +path: wrongpath`, + expectedErr: "invalid configuration: path must start with /", + }, + { + config: ` +source: http +port: 8080 path: /test auth_type: basic_auth`, expectedErr: "invalid configuration: basic_auth is required", From 075bb92e75a35f2d75c24a88afae4d9a4906b91a Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 11:48:12 +0200 Subject: [PATCH 13/30] comment for now chunk_size param --- pkg/acquisition/modules/http/http.go | 10 ++++++---- pkg/acquisition/modules/http/http_test.go | 11 ----------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 15f1958aea8..9f2a6197f57 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -38,6 +38,7 @@ var linesRead = prometheus.NewCounterVec( type HttpConfiguration struct { //IPFilter []string `yaml:"ip_filter"` + //ChunkSize *int64 `yaml:"chunk_size"` Port int `yaml:"port"` Path string `yaml:"path"` AuthType string `yaml:"auth_type"` @@ -47,7 +48,6 @@ type HttpConfiguration struct { CustomStatusCode *int `yaml:"custom_status_code"` CustomHeaders *map[string]string `yaml:"custom_headers"` MaxBodySize *int64 `yaml:"max_body_size"` - ChunkSize *int64 `yaml:"chunk_size"` Timeout *time.Duration `yaml:"timeout"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -138,9 +138,11 @@ func (hc *HttpConfiguration) Validate() error { return errors.New("max_body_size must be positive") } - if hc.ChunkSize != nil && *hc.ChunkSize <= 0 { - return errors.New("chunk_size must be positive") - } + /* + if hc.ChunkSize != nil && *hc.ChunkSize <= 0 { + return errors.New("chunk_size must be positive") + } + */ if hc.CustomStatusCode != nil { statusText := http.StatusText(*hc.CustomStatusCode) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 29221b5dc49..e7bc162b9e0 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -158,17 +158,6 @@ source: http port: 8080 path: /test auth_type: headers -headers: - key: value -chunk_size: 0`, - expectedErr: "invalid configuration: chunk_size must be positive", - }, - { - config: ` -source: http -port: 8080 -path: /test -auth_type: headers headers: key: value timeout: toto`, From bdb0a40a1de0746fbaef10bd79378e43d03f27f4 Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 12:58:56 +0200 Subject: [PATCH 14/30] handle ndjson + adapt tests to code and not the inverse --- pkg/acquisition/modules/http/http.go | 66 +++++++++++--- pkg/acquisition/modules/http/http_test.go | 104 +++++++++++++++++++--- 2 files changed, 145 insertions(+), 25 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 9f2a6197f57..2cdde786f44 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/json" "errors" "fmt" "io" @@ -258,7 +259,19 @@ func ReadBody(r *http.Request) ([]byte, error) { return io.ReadAll(gReader) } - return io.ReadAll(r.Body) + decoder := json.NewDecoder(r.Body) + var body []byte + for { + var message json.RawMessage + if err := decoder.Decode(&message); err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("failed to decode: %w", err) + } + body = append(body, message...) + } + return body, nil } func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc *HttpConfiguration, out chan types.Event, t *tomb.Tomb) error { @@ -267,22 +280,37 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * return fmt.Errorf("body size exceeds max body size: %d > %d", r.ContentLength, *hc.MaxBodySize) } - body, err := ReadBody(r) - defer r.Body.Close() - if err != nil { - w.WriteHeader(http.StatusBadRequest) - return fmt.Errorf("failed to read body: %w", err) - } - h.logger.Tracef("body received: %+v", string(body)) - srcHost, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { return err } - t.Go(func() error { + defer r.Body.Close() + + reader := r.Body + + if r.Header.Get("Content-Encoding") == "gzip" { + reader, err = gzip.NewReader(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer reader.Close() + } + + decoder := json.NewDecoder(reader) + for { + var message json.RawMessage + if err := decoder.Decode(&message); err != nil { + if err == io.EOF { + break + } + w.WriteHeader(http.StatusBadRequest) + return fmt.Errorf("failed to decode: %w", err) + } + line := types.Line{ - Raw: string(body), + Raw: string(message), Src: srcHost, Time: time.Now().UTC(), Labels: hc.Labels, @@ -306,9 +334,21 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * linesRead.With(prometheus.Labels{"path": hc.Path}).Inc() } + h.logger.Tracef("line to send: %+v", line) out <- evt - return nil - }) + } + + //body, err := ReadBody(r) + //defer r.Body.Close() + //if err != nil { + // w.WriteHeader(http.StatusBadRequest) + // return fmt.Errorf("failed to read body: %w", err) + //} + //h.logger.Tracef("body received: %+v", string(body)) + // + //t.Go(func() error { + // return nil + //}) return nil } diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index e7bc162b9e0..b2efebf24b6 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -422,6 +422,9 @@ headers: time.Sleep(1 * time.Second) rawEvt := `{"test": "test"}` + errChan := make(chan error) + go assertEvent(out, rawEvt, errChan) + client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) if err != nil { @@ -436,7 +439,10 @@ headers: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertFirstEvent(out, t, rawEvt) + err = <-errChan + if err != nil { + t.Fatalf("error: %s", err) + } h.Server.Close() tomb.Kill(nil) @@ -459,6 +465,9 @@ custom_headers: time.Sleep(1 * time.Second) rawEvt := `{"test": "test"}` + errChan := make(chan error) + go assertEvent(out, rawEvt, errChan) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) if err != nil { t.Fatalf("unable to create http request: %s", err) @@ -477,7 +486,10 @@ custom_headers: t.Fatalf("expected header 'success' to be 'true', got '%s'", resp.Header.Get("Success")) } - assertFirstEvent(out, t, rawEvt) + err = <-errChan + if err != nil { + t.Fatalf("error: %s", err) + } h.Server.Close() tomb.Kill(nil) @@ -500,28 +512,34 @@ func (sr *slowReader) Read(p []byte) (int, error) { return n, nil } -func assertFirstEvent(out chan types.Event, t *testing.T, expected string) { +func assertEvent(out chan types.Event, expected string, errChan chan error) { readLines := []types.Event{} select { case event := <-out: readLines = append(readLines, event) case <-time.After(2 * time.Second): - break + errChan <- fmt.Errorf("timeout waiting for event") + return } if len(readLines) != 1 { - t.Fatalf("expected 1 line, got %d", len(readLines)) + errChan <- fmt.Errorf("expected 1 line, got %d", len(readLines)) + return } if readLines[0].Line.Raw != expected { - t.Fatalf(`expected %s, got '%+v'`, expected, readLines[0].Line) + errChan <- fmt.Errorf(`expected %s, got '%+v'`, expected, readLines[0].Line.Raw) + return } if readLines[0].Line.Src != "127.0.0.1" { - t.Fatalf("expected '127.0.0.1', got '%s'", readLines[0].Line.Src) + errChan <- fmt.Errorf("expected '127.0.0.1', got '%s'", readLines[0].Line.Src) + return } if readLines[0].Line.Module != "http" { - t.Fatalf("expected 'http', got '%s'", readLines[0].Line.Module) + errChan <- fmt.Errorf("expected 'http', got '%s'", readLines[0].Line.Module) + return } + errChan <- nil } func TestStreamingAcquisitionTimeout(t *testing.T) { @@ -624,6 +642,8 @@ tls: } rawEvt := `{"test": "test"}` + errChan := make(chan error) + go assertEvent(out, rawEvt, errChan) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) if err != nil { @@ -638,7 +658,10 @@ tls: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertFirstEvent(out, t, rawEvt) + err = <-errChan + if err != nil { + t.Fatalf("error: %s", err) + } h.Server.Close() tomb.Kill(nil) @@ -685,6 +708,9 @@ tls: } rawEvt := `{"test": "test"}` + errChan := make(chan error) + go assertEvent(out, rawEvt, errChan) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) if err != nil { t.Fatalf("unable to create http request: %s", err) @@ -699,7 +725,10 @@ tls: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertFirstEvent(out, t, rawEvt) + err = <-errChan + if err != nil { + t.Fatalf("error: %s", err) + } h.Server.Close() tomb.Kill(nil) @@ -719,10 +748,13 @@ headers: time.Sleep(1 * time.Second) rawEvt := `{"test": "test"}` + errChan := make(chan error) + go assertEvent(out, rawEvt, errChan) + go assertEvent(out, rawEvt, errChan) // send gzipped compressed data client := &http.Client{} - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(fmt.Sprintf("%s\n%s", rawEvt, rawEvt))) if err != nil { t.Fatalf("unable to create http request: %s", err) } @@ -749,7 +781,55 @@ headers: t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) } - assertFirstEvent(out, t, rawEvt) + err = <-errChan + if err != nil { + t.Fatalf("error: %s", err) + } + + h.Server.Close() + tomb.Kill(nil) + tomb.Wait() +} + +func TestStreamingAcquisitionNDJson(t *testing.T) { + h := &HTTPSource{} + out, tomb := SetupAndRunHTTPSource(t, h, []byte(` +source: http +port: 8080 +path: /test +auth_type: headers +headers: + key: test`)) + + time.Sleep(1 * time.Second) + rawEvt := `{"test": "test"}` + + errChan := make(chan error) + go assertEvent(out, rawEvt, errChan) + go assertEvent(out, rawEvt, errChan) + + client := &http.Client{} + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(fmt.Sprintf("%s\n%s\n", rawEvt, rawEvt))) + + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + + req.Header.Add("Key", "test") + req.Header.Add("Content-Type", "application/x-ndjson") + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("unable to post http request: %s", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + err = <-errChan + if err != nil { + t.Fatalf("error: %s", err) + } h.Server.Close() tomb.Kill(nil) From da457ee4510e8fbe5b38f0e21ba5469392757e80 Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 13:00:14 +0200 Subject: [PATCH 15/30] remove unused param in func --- pkg/acquisition/modules/http/http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 2cdde786f44..255317c4ec0 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -274,7 +274,7 @@ func ReadBody(r *http.Request) ([]byte, error) { return body, nil } -func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc *HttpConfiguration, out chan types.Event, t *tomb.Tomb) error { +func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc *HttpConfiguration, out chan types.Event) error { if hc.MaxBodySize != nil && r.ContentLength > *hc.MaxBodySize { w.WriteHeader(http.StatusRequestEntityTooLarge) return fmt.Errorf("body size exceeds max body size: %d > %d", r.ContentLength, *hc.MaxBodySize) @@ -366,7 +366,7 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { http.Error(w, "Unauthorized", http.StatusUnauthorized) return } - err := h.processRequest(w, r, &h.Config, out, t) + err := h.processRequest(w, r, &h.Config, out) if err != nil { h.logger.Errorf("failed to process request: %s", err) return From 5de442a369fb7db47d11ef9b965f2a5316a99dfe Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 15:16:36 +0200 Subject: [PATCH 16/30] forgotten file --- pkg/acquisition/http.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 pkg/acquisition/http.go diff --git a/pkg/acquisition/http.go b/pkg/acquisition/http.go new file mode 100644 index 00000000000..59745772b62 --- /dev/null +++ b/pkg/acquisition/http.go @@ -0,0 +1,12 @@ +//go:build !no_datasource_http + +package acquisition + +import ( + httpacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/http" +) + +//nolint:gochecknoinits +func init() { + registerDataSource("http", func() DataSource { return &httpacquisition.HTTPSource{} }) +} From a53e2a96dc51f6302dd6c5765c6c490314d03a87 Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 15:39:28 +0200 Subject: [PATCH 17/30] fix comments --- pkg/acquisition/modules/http/http.go | 25 ++++---- pkg/acquisition/modules/http/http_test.go | 71 ++++++++++------------- 2 files changed, 42 insertions(+), 54 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 255317c4ec0..9e9f63074bd 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -40,7 +40,7 @@ var linesRead = prometheus.NewCounterVec( type HttpConfiguration struct { //IPFilter []string `yaml:"ip_filter"` //ChunkSize *int64 `yaml:"chunk_size"` - Port int `yaml:"port"` + ListenAddr string `yaml:"listen_addr"` Path string `yaml:"path"` AuthType string `yaml:"auth_type"` BasicAuth *BasicAuthConfig `yaml:"basic_auth"` @@ -91,11 +91,12 @@ func (h *HTTPSource) UnmarshalConfig(yamlConfig []byte) error { } func (hc *HttpConfiguration) Validate() error { - if hc.Port == 0 { - return errors.New("port is required") + if hc.ListenAddr == "" { + return errors.New("listen_addr is required") } + if hc.Path == "" { - return errors.New("path is required") + hc.Path = "/" } if hc.Path[0] != '/' { return errors.New("path must start with /") @@ -121,9 +122,7 @@ func (hc *HttpConfiguration) Validate() error { return errors.New("ca_cert is required") } default: - if hc.TLS == nil { - return errors.New("at least one of tls or auth_type is required") - } + return errors.New("invalid auth_type: must be one of basic_auth, headers, mtls") } if hc.TLS != nil { @@ -362,13 +361,13 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { return } if err := authorizeRequest(r, &h.Config); err != nil { - h.logger.Errorf("failed to authorize request: %s", err) + h.logger.Errorf("failed to authorize request from '%s': %s", r.RemoteAddr, err) http.Error(w, "Unauthorized", http.StatusUnauthorized) return } err := h.processRequest(w, r, &h.Config, out) if err != nil { - h.logger.Errorf("failed to process request: %s", err) + h.logger.Errorf("failed to process request from '%s': %s", r.RemoteAddr, err) return } @@ -387,7 +386,7 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { }) h.Server = &http.Server{ - Addr: fmt.Sprintf(":%d", h.Config.Port), + Addr: h.Config.ListenAddr, Handler: mux, } @@ -407,13 +406,13 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/http/server") if h.Config.TLS != nil { - h.logger.Infof("start https server on port %d", h.Config.Port) + h.logger.Infof("start https server on %s", h.Config.ListenAddr) err := h.Server.ListenAndServeTLS(h.Config.TLS.ServerCert, h.Config.TLS.ServerKey) if err != nil && err != http.ErrServerClosed { return fmt.Errorf("https server failed: %w", err) } } else { - h.logger.Infof("start http server on port %d", h.Config.Port) + h.logger.Infof("start http server on %s", h.Config.ListenAddr) err := h.Server.ListenAndServe() if err != nil && err != http.ErrServerClosed { return fmt.Errorf("http server failed: %w", err) @@ -436,7 +435,7 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { } func (h *HTTPSource) StreamingAcquisition(ctx context.Context, out chan types.Event, t *tomb.Tomb) error { - h.logger.Debugf("start http server on port %d", h.Config.Port) + h.logger.Debugf("start http server on %s", h.Config.ListenAddr) t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/http/live") diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index b2efebf24b6..590248fd651 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -32,31 +32,19 @@ func TestConfigure(t *testing.T) { { config: ` foobar: bla`, - expectedErr: "invalid configuration: port is required", + expectedErr: "invalid configuration: listen_addr is required", }, { config: ` source: http -port: aa`, - expectedErr: "cannot parse http datasource configuration: yaml: unmarshal errors:\n line 3: cannot unmarshal !!str `aa` into int", - }, - { - config: ` -source: http -port: 8080`, - expectedErr: "invalid configuration: path is required", - }, - { - config: ` -source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: wrongpath`, expectedErr: "invalid configuration: path must start with /", }, { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth`, expectedErr: "invalid configuration: basic_auth is required", @@ -64,7 +52,7 @@ auth_type: basic_auth`, { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers`, expectedErr: "invalid configuration: headers is required", @@ -72,7 +60,7 @@ auth_type: headers`, { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth basic_auth: @@ -82,7 +70,7 @@ basic_auth: { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth basic_auth: @@ -92,7 +80,7 @@ basic_auth: { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers:`, @@ -101,15 +89,15 @@ headers:`, { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: toto`, - expectedErr: "invalid configuration: at least one of tls or auth_type is required", + expectedErr: "invalid configuration: invalid auth_type: must be one of basic_auth, headers, mtls", }, { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -121,7 +109,7 @@ tls: { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -133,7 +121,7 @@ tls: { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: mtls tls: @@ -144,7 +132,7 @@ tls: { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -155,7 +143,7 @@ max_body_size: 0`, { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -166,7 +154,7 @@ timeout: toto`, { config: ` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -199,7 +187,7 @@ func TestUnmarshalConfig(t *testing.T) { h := HTTPSource{} err := h.UnmarshalConfig([]byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: 15 auth_type: headers`)) cstest.AssertErrorMessage(t, err, "cannot parse http datasource configuration: yaml: line 4: found a tab character that violates indentation") @@ -254,7 +242,7 @@ func TestStreamingAcquisitionWrongHTTPMethod(t *testing.T) { h := &HTTPSource{} _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth basic_auth: @@ -281,7 +269,7 @@ func TestStreamingAcquisitionUnknownPath(t *testing.T) { h := &HTTPSource{} _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth basic_auth: @@ -308,7 +296,7 @@ func TestStreamingAcquisitionBasicAuth(t *testing.T) { h := &HTTPSource{} _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth basic_auth: @@ -349,7 +337,7 @@ func TestStreamingAcquisitionBadHeaders(t *testing.T) { h := &HTTPSource{} _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -381,7 +369,7 @@ func TestStreamingAcquisitionMaxBodySize(t *testing.T) { h := &HTTPSource{} _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -413,7 +401,7 @@ func TestStreamingAcquisitionSuccess(t *testing.T) { h := &HTTPSource{} out, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -453,7 +441,7 @@ func TestStreamingAcquisitionCustomStatusCodeAndCustomHeaders(t *testing.T) { h := &HTTPSource{} out, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -546,7 +534,7 @@ func TestStreamingAcquisitionTimeout(t *testing.T) { h := &HTTPSource{} _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -585,7 +573,8 @@ func TestStreamingAcquisitionTLSHTTPRequest(t *testing.T) { h := &HTTPSource{} _, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 +auth_type: mtls path: /test tls: server_cert: testdata/server.crt @@ -612,7 +601,7 @@ func TestStreamingAcquisitionTLSWithHeadersAuthSuccess(t *testing.T) { h := &HTTPSource{} out, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -672,7 +661,7 @@ func TestStreamingAcquisitionMTLS(t *testing.T) { h := &HTTPSource{} out, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: mtls tls: @@ -739,7 +728,7 @@ func TestStreamingAcquisitionGzipData(t *testing.T) { h := &HTTPSource{} out, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -795,7 +784,7 @@ func TestStreamingAcquisitionNDJson(t *testing.T) { h := &HTTPSource{} out, tomb := SetupAndRunHTTPSource(t, h, []byte(` source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: From 6004557767d568fb0974520ff99455ccbfb5804c Mon Sep 17 00:00:00 2001 From: he2ss Date: Mon, 28 Oct 2024 10:10:01 +0100 Subject: [PATCH 18/30] fix lint --- pkg/acquisition/modules/http/http_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 590248fd651..b0ab86ba34c 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net/http" @@ -507,7 +508,7 @@ func assertEvent(out chan types.Event, expected string, errChan chan error) { case event := <-out: readLines = append(readLines, event) case <-time.After(2 * time.Second): - errChan <- fmt.Errorf("timeout waiting for event") + errChan <- errors.New("timeout waiting for event") return } From 81416093757c057fb4ab4cdf36fb663db35c73c1 Mon Sep 17 00:00:00 2001 From: he2ss Date: Mon, 28 Oct 2024 15:18:24 +0100 Subject: [PATCH 19/30] update metrics + fix tests --- pkg/acquisition/modules/http/http.go | 46 +----- pkg/acquisition/modules/http/http_test.go | 170 +++++++++++++++------- 2 files changed, 121 insertions(+), 95 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 9e9f63074bd..c3a199bcffb 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -35,7 +35,7 @@ var linesRead = prometheus.NewCounterVec( Name: "cs_httpsource_hits_total", Help: "Total lines that were read from http source", }, - []string{"path"}) + []string{"path", "src"}) type HttpConfiguration struct { //IPFilter []string `yaml:"ip_filter"` @@ -248,31 +248,6 @@ func authorizeRequest(r *http.Request, hc *HttpConfiguration) error { return nil } -func ReadBody(r *http.Request) ([]byte, error) { - if r.Header.Get("Content-Encoding") == "gzip" { - gReader, err := gzip.NewReader(r.Body) - if err != nil { - return nil, fmt.Errorf("failed to create gzip reader: %w", err) - } - defer gReader.Close() - return io.ReadAll(gReader) - } - - decoder := json.NewDecoder(r.Body) - var body []byte - for { - var message json.RawMessage - if err := decoder.Decode(&message); err != nil { - if err == io.EOF { - break - } - return nil, fmt.Errorf("failed to decode: %w", err) - } - body = append(body, message...) - } - return body, nil -} - func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc *HttpConfiguration, out chan types.Event) error { if hc.MaxBodySize != nil && r.ContentLength > *hc.MaxBodySize { w.WriteHeader(http.StatusRequestEntityTooLarge) @@ -300,6 +275,7 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * decoder := json.NewDecoder(reader) for { var message json.RawMessage + if err := decoder.Decode(&message); err != nil { if err == io.EOF { break @@ -329,26 +305,16 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * Unmarshaled: make(map[string]interface{}), } - if h.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"path": hc.Path}).Inc() + if h.metricsLevel == configuration.METRICS_AGGREGATE { + linesRead.With(prometheus.Labels{"path": hc.Path, "src": ""}).Inc() + } else if h.metricsLevel == configuration.METRICS_FULL { + linesRead.With(prometheus.Labels{"path": hc.Path, "src": srcHost}).Inc() } h.logger.Tracef("line to send: %+v", line) out <- evt } - //body, err := ReadBody(r) - //defer r.Body.Close() - //if err != nil { - // w.WriteHeader(http.StatusBadRequest) - // return fmt.Errorf("failed to read body: %w", err) - //} - //h.logger.Tracef("body received: %+v", string(body)) - // - //t.Go(func() error { - // return nil - //}) - return nil } diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index b0ab86ba34c..aac7a5b7869 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -16,6 +16,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/go-cs-lib/cstest" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" ) @@ -221,12 +222,12 @@ func TestGetName(t *testing.T) { } } -func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte) (chan types.Event, *tomb.Tomb) { +func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLevel int) (chan types.Event, *tomb.Tomb) { ctx := context.Background() subLogger := log.WithFields(log.Fields{ "type": "http", }) - err := h.Configure(config, subLogger, 0) + err := h.Configure(config, subLogger, metricLevel) if err != nil { t.Fatalf("unable to configure http source: %s", err) } @@ -236,6 +237,11 @@ func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte) (chan typ if err != nil { t.Fatalf("unable to start streaming acquisition: %s", err) } + + for _, metric := range h.GetMetrics() { + prometheus.Register(metric) + } + return out, &tomb } @@ -248,7 +254,7 @@ path: /test auth_type: basic_auth basic_auth: username: test - password: test`)) + password: test`), 0) time.Sleep(1 * time.Second) @@ -275,7 +281,7 @@ path: /test auth_type: basic_auth basic_auth: username: test - password: test`)) + password: test`), 0) time.Sleep(1 * time.Second) @@ -302,7 +308,7 @@ path: /test auth_type: basic_auth basic_auth: username: test - password: test`)) + password: test`), 0) time.Sleep(1 * time.Second) @@ -342,7 +348,7 @@ listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: - key: test`)) + key: test`), 0) time.Sleep(1 * time.Second) @@ -375,7 +381,7 @@ path: /test auth_type: headers headers: key: test -max_body_size: 5`)) +max_body_size: 5`), 0) time.Sleep(1 * time.Second) @@ -406,13 +412,13 @@ listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: - key: test`)) + key: test`), 2) time.Sleep(1 * time.Second) rawEvt := `{"test": "test"}` errChan := make(chan error) - go assertEvent(out, rawEvt, errChan) + go assertEvents(out, []string{rawEvt}, errChan) client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) @@ -433,6 +439,8 @@ headers: t.Fatalf("error: %s", err) } + assertMetrics(t, h.GetMetrics(), 1) + h.Server.Close() tomb.Kill(nil) tomb.Wait() @@ -449,13 +457,13 @@ headers: key: test custom_status_code: 201 custom_headers: - success: true`)) + success: true`), 2) time.Sleep(1 * time.Second) rawEvt := `{"test": "test"}` errChan := make(chan error) - go assertEvent(out, rawEvt, errChan) + go assertEvents(out, []string{rawEvt}, errChan) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) if err != nil { @@ -480,6 +488,8 @@ custom_headers: t.Fatalf("error: %s", err) } + assertMetrics(t, h.GetMetrics(), 1) + h.Server.Close() tomb.Kill(nil) tomb.Wait() @@ -501,32 +511,37 @@ func (sr *slowReader) Read(p []byte) (int, error) { return n, nil } -func assertEvent(out chan types.Event, expected string, errChan chan error) { +func assertEvents(out chan types.Event, expected []string, errChan chan error) { readLines := []types.Event{} - select { - case event := <-out: - readLines = append(readLines, event) - case <-time.After(2 * time.Second): - errChan <- errors.New("timeout waiting for event") - return + for i := 0; i < len(expected); i++ { + select { + case event := <-out: + readLines = append(readLines, event) + case <-time.After(2 * time.Second): + errChan <- errors.New("timeout waiting for event") + return + } } - if len(readLines) != 1 { - errChan <- fmt.Errorf("expected 1 line, got %d", len(readLines)) - return - } - if readLines[0].Line.Raw != expected { - errChan <- fmt.Errorf(`expected %s, got '%+v'`, expected, readLines[0].Line.Raw) - return - } - if readLines[0].Line.Src != "127.0.0.1" { - errChan <- fmt.Errorf("expected '127.0.0.1', got '%s'", readLines[0].Line.Src) + if len(readLines) != len(expected) { + errChan <- fmt.Errorf("expected %d lines, got %d", len(expected), len(readLines)) return } - if readLines[0].Line.Module != "http" { - errChan <- fmt.Errorf("expected 'http', got '%s'", readLines[0].Line.Module) - return + + for i, evt := range readLines { + if evt.Line.Raw != expected[i] { + errChan <- fmt.Errorf(`expected %s, got '%+v'`, expected, evt.Line.Raw) + return + } + if evt.Line.Src != "127.0.0.1" { + errChan <- fmt.Errorf("expected '127.0.0.1', got '%s'", evt.Line.Src) + return + } + if evt.Line.Module != "http" { + errChan <- fmt.Errorf("expected 'http', got '%s'", evt.Line.Module) + return + } } errChan <- nil } @@ -540,7 +555,7 @@ path: /test auth_type: headers headers: key: test -timeout: 1s`)) +timeout: 1s`), 0) time.Sleep(1 * time.Second) @@ -580,7 +595,7 @@ path: /test tls: server_cert: testdata/server.crt server_key: testdata/server.key - ca_cert: testdata/ca.crt`)) + ca_cert: testdata/ca.crt`), 0) time.Sleep(1 * time.Second) @@ -610,7 +625,7 @@ headers: tls: server_cert: testdata/server.crt server_key: testdata/server.key -`)) +`), 0) time.Sleep(1 * time.Second) @@ -633,7 +648,7 @@ tls: rawEvt := `{"test": "test"}` errChan := make(chan error) - go assertEvent(out, rawEvt, errChan) + go assertEvents(out, []string{rawEvt}, errChan) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) if err != nil { @@ -653,6 +668,8 @@ tls: t.Fatalf("error: %s", err) } + assertMetrics(t, h.GetMetrics(), 0) + h.Server.Close() tomb.Kill(nil) tomb.Wait() @@ -668,7 +685,7 @@ auth_type: mtls tls: server_cert: testdata/server.crt server_key: testdata/server.key - ca_cert: testdata/ca.crt`)) + ca_cert: testdata/ca.crt`), 0) time.Sleep(1 * time.Second) @@ -699,7 +716,7 @@ tls: rawEvt := `{"test": "test"}` errChan := make(chan error) - go assertEvent(out, rawEvt, errChan) + go assertEvents(out, []string{rawEvt}, errChan) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) if err != nil { @@ -720,6 +737,8 @@ tls: t.Fatalf("error: %s", err) } + assertMetrics(t, h.GetMetrics(), 0) + h.Server.Close() tomb.Kill(nil) tomb.Wait() @@ -733,35 +752,35 @@ listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: - key: test`)) + key: test`), 2) time.Sleep(1 * time.Second) rawEvt := `{"test": "test"}` errChan := make(chan error) - go assertEvent(out, rawEvt, errChan) - go assertEvent(out, rawEvt, errChan) - - // send gzipped compressed data - client := &http.Client{} - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(fmt.Sprintf("%s\n%s", rawEvt, rawEvt))) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } - req.Header.Add("Key", "test") - req.Header.Add("Content-Encoding", "gzip") - req.Header.Add("Content-Type", "application/json") + go assertEvents(out, []string{rawEvt, rawEvt}, errChan) var b strings.Builder gz := gzip.NewWriter(&b) if _, err := gz.Write([]byte(rawEvt)); err != nil { t.Fatalf("unable to write gzipped data: %s", err) } + if _, err := gz.Write([]byte(rawEvt)); err != nil { + t.Fatalf("unable to write gzipped data: %s", err) + } if err := gz.Close(); err != nil { t.Fatalf("unable to close gzip writer: %s", err) } - req.Body = io.NopCloser(strings.NewReader(b.String())) - req.ContentLength = int64(b.Len()) + + // send gzipped compressed data + client := &http.Client{} + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(b.String())) + if err != nil { + t.Fatalf("unable to create http request: %s", err) + } + req.Header.Add("Key", "test") + req.Header.Add("Content-Encoding", "gzip") + req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { @@ -776,6 +795,8 @@ headers: t.Fatalf("error: %s", err) } + assertMetrics(t, h.GetMetrics(), 2) + h.Server.Close() tomb.Kill(nil) tomb.Wait() @@ -789,14 +810,13 @@ listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: - key: test`)) + key: test`), 2) time.Sleep(1 * time.Second) rawEvt := `{"test": "test"}` errChan := make(chan error) - go assertEvent(out, rawEvt, errChan) - go assertEvent(out, rawEvt, errChan) + go assertEvents(out, []string{rawEvt, rawEvt}, errChan) client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(fmt.Sprintf("%s\n%s\n", rawEvt, rawEvt))) @@ -821,7 +841,47 @@ headers: t.Fatalf("error: %s", err) } + assertMetrics(t, h.GetMetrics(), 2) + h.Server.Close() tomb.Kill(nil) tomb.Wait() } + +func assertMetrics(t *testing.T, metrics []prometheus.Collector, expected int) { + promMetrics, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatalf("unable to gather metrics: %s", err) + } + isExist := false + for _, metricFamily := range promMetrics { + if metricFamily.GetName() == "cs_httpsource_hits_total" { + isExist = true + if len(metricFamily.GetMetric()) != 1 { + t.Fatalf("expected 1 metricFamily, got %d", len(metricFamily.GetMetric())) + } + for _, metric := range metricFamily.GetMetric() { + if metric.GetCounter().GetValue() != float64(expected) { + t.Fatalf("expected %d, got %f", expected, metric.GetCounter().GetValue()) + } + labels := metric.GetLabel() + if len(labels) != 2 { + t.Fatalf("expected 2 label, got %d", len(labels)) + } + if labels[0].GetName() != "path" || labels[0].GetValue() != "/test" { + t.Fatalf("expected label path:/test, got %s:%s", labels[0].GetName(), labels[0].GetValue()) + } + if labels[1].GetName() != "src" || labels[1].GetValue() != "127.0.0.1" { + t.Fatalf("expected label src:127.0.0.1, got %s:%s", labels[1].GetName(), labels[1].GetValue()) + } + } + } + } + if !isExist && expected > 0 { + t.Fatalf("expected metric cs_httpsource_hits_total not found") + } + + for _, metric := range metrics { + metric.(*prometheus.CounterVec).Reset() + } +} From 168b4522680627fdd572e9f872cfd361f41b3dc5 Mon Sep 17 00:00:00 2001 From: he2ss Date: Mon, 28 Oct 2024 15:54:54 +0100 Subject: [PATCH 20/30] fix laurence comments --- pkg/acquisition/modules/http/http.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index c3a199bcffb..dd42df90a94 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -118,7 +118,7 @@ func (hc *HttpConfiguration) Validate() error { return errors.New("headers is required") } case "mtls": - if hc.TLS == nil || hc.TLS != nil && hc.TLS.CaCert == "" { + if hc.TLS == nil || hc.TLS.CaCert == "" { return errors.New("ca_cert is required") } default: @@ -219,7 +219,15 @@ func (hc *HttpConfiguration) NewTLSConfig() (*tls.Config, error) { if err != nil { return nil, fmt.Errorf("failed to read ca cert: %w", err) } - caCertPool := x509.NewCertPool() + + caCertPool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("failed to load system cert pool: %w", err) + } + + if caCertPool == nil { + caCertPool = x509.NewCertPool() + } caCertPool.AppendCertsFromPEM(caCert) tlsConfig.ClientCAs = caCertPool tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert From d5a485cc3e70b7918a10efa14b637bdf69f5f062 Mon Sep 17 00:00:00 2001 From: he2ss Date: Mon, 28 Oct 2024 16:03:19 +0100 Subject: [PATCH 21/30] fix typo --- pkg/acquisition/modules/http/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index dd42df90a94..71ec258164b 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -181,7 +181,7 @@ func (h *HTTPSource) GetName() string { return dataSourceName } -func (h *HTTPSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { +func (h *HTTPSource) OneShotAcquisition(ctx context.Context, out chan types.Event, t *tomb.Tomb) error { return fmt.Errorf("%s datasource does not support one-shot acquisition", dataSourceName) } From 750a6b62c08a34bec5e33a2bee1061e7da3e0456 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 28 Oct 2024 16:44:48 +0100 Subject: [PATCH 22/30] constructor for types.Event --- pkg/acquisition/modules/cloudwatch/cloudwatch.go | 4 +--- pkg/acquisition/modules/docker/docker.go | 16 ++++++++-------- pkg/acquisition/modules/file/file.go | 10 +++++----- pkg/acquisition/modules/http/http.go | 11 ++++------- pkg/acquisition/modules/journalctl/journalctl.go | 12 +++++------- pkg/acquisition/modules/kafka/kafka.go | 11 ++++------- pkg/acquisition/modules/kinesis/kinesis.go | 11 ++++------- .../modules/kubernetesaudit/k8s_audit.go | 12 +++++------- pkg/acquisition/modules/loki/loki.go | 16 +++++----------- pkg/acquisition/modules/s3/s3.go | 11 ++++------- pkg/acquisition/modules/syslog/syslog.go | 10 +++++----- .../modules/wineventlog/wineventlog_windows.go | 6 +++++- pkg/types/event.go | 13 +++++++++++++ 13 files changed, 68 insertions(+), 75 deletions(-) diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index b028c79db20..3c027c41702 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -710,7 +710,7 @@ func (cw *CloudwatchSource) CatLogStream(ctx context.Context, cfg *LogStreamTail func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig) (types.Event, error) { l := types.Line{} - evt := types.Event{} + evt := types.MakeEvent(cfg.ExpectMode == types.TIMEMACHINE) if log.Message == nil { return evt, errors.New("nil message") } @@ -728,8 +728,6 @@ func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig) evt.Line = l evt.Process = true evt.Type = types.LOG - evt.ExpectMode = cfg.ExpectMode - evt.Unmarshaled = make(map[string]interface{}) cfg.logger.Debugf("returned event labels : %+v", evt.Line.Labels) return evt, nil } diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index d41d6479161..2689d5ab9f0 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -334,7 +334,10 @@ func (d *DockerSource) OneShotAcquisition(ctx context.Context, out chan types.Ev if d.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"source": containerConfig.Name}).Inc() } - evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} + evt := types.MakeEvent(true) + evt.Line = l + evt.Process = true + evt.Type = types.LOG out <- evt d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw) } @@ -579,13 +582,10 @@ func (d *DockerSource) TailDocker(ctx context.Context, container *ContainerConfi l.Src = container.Name l.Process = true l.Module = d.GetName() - var evt types.Event - evt.Unmarshaled = make(map[string]interface{}) - if !d.Config.UseTimeMachine { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} - } else { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} - } + evt := types.MakeEvent(d.Config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG linesRead.With(prometheus.Labels{"source": container.Name}).Inc() outChan <- evt d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw) diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index bd87f69d637..027bb323942 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -621,11 +621,11 @@ func (f *FileSource) tailFile(out chan types.Event, t *tomb.Tomb, tail *tail.Tai // we're tailing, it must be real time logs logger.Debugf("pushing %+v", l) - expectMode := types.LIVE - if f.config.UseTimeMachine { - expectMode = types.TIMEMACHINE - } - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: expectMode, Unmarshaled: make(map[string]interface{})} + evt := types.MakeEvent(f.config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG + out <- evt } } } diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 71ec258164b..811396b5299 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -305,13 +305,10 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * line.Src = hc.Path } - evt := types.Event{ - Line: line, - Process: true, - Type: types.LOG, - ExpectMode: types.LIVE, - Unmarshaled: make(map[string]interface{}), - } + evt := types.MakeEvent(h.Config.UseTimeMachine) + evt.Line = line + evt.Process = true + evt.Type = types.LOG if h.metricsLevel == configuration.METRICS_AGGREGATE { linesRead.With(prometheus.Labels{"path": hc.Path, "src": ""}).Inc() diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index b6162fa50c5..ae4c32db81a 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -136,13 +136,11 @@ func (j *JournalCtlSource) runJournalCtl(ctx context.Context, out chan types.Eve if j.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"source": j.src}).Inc() } - var evt types.Event - evt.Unmarshaled = make(map[string]interface{}) - if !j.config.UseTimeMachine { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} - } else { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} - } + + evt := types.MakeEvent(j.config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG out <- evt case stderrLine := <-stderrChan: logger.Warnf("Got stderr message : %s", stderrLine) diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 002850864d4..6c44470601f 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -173,13 +173,10 @@ func (k *KafkaSource) ReadMessage(ctx context.Context, out chan types.Event) err if k.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"topic": k.Config.Topic}).Inc() } - var evt types.Event - evt.Unmarshaled = make(map[string]interface{}) - if !k.Config.UseTimeMachine { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} - } else { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} - } + evt := types.MakeEvent(k.Config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG out <- evt } } diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index d1700e4baf4..632395c1eb2 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -322,13 +322,10 @@ func (k *KinesisSource) ParseAndPushRecords(records []*kinesis.Record, out chan } else { l.Src = k.Config.StreamName } - var evt types.Event - evt.Unmarshaled = make(map[string]interface{}) - if !k.Config.UseTimeMachine { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} - } else { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} - } + evt := types.MakeEvent(k.Config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG out <- evt } } diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index 64769228e31..9272e6986cc 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -207,12 +207,10 @@ func (ka *KubernetesAuditSource) webhookHandler(w http.ResponseWriter, r *http.R Process: true, Module: ka.GetName(), } - ka.outChan <- types.Event{ - Line: l, - Process: true, - Type: types.LOG, - ExpectMode: types.LIVE, - Unmarshaled: make(map[string]interface{}), - } + evt := types.MakeEvent(ka.config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG + ka.outChan <- evt } } diff --git a/pkg/acquisition/modules/loki/loki.go b/pkg/acquisition/modules/loki/loki.go index ba9778fec12..88a9974ca07 100644 --- a/pkg/acquisition/modules/loki/loki.go +++ b/pkg/acquisition/modules/loki/loki.go @@ -307,17 +307,11 @@ func (l *LokiSource) readOneEntry(entry lokiclient.Entry, labels map[string]stri if l.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"source": l.Config.URL}).Inc() } - expectMode := types.LIVE - if l.Config.UseTimeMachine { - expectMode = types.TIMEMACHINE - } - out <- types.Event{ - Line: ll, - Process: true, - Type: types.LOG, - ExpectMode: expectMode, - Unmarshaled: make(map[string]interface{}), - } + evt := types.MakeEvent(l.Config.UseTimeMachine) + evt.Line = ll + evt.Process = true + evt.Type = types.LOG + out <- evt } func (l *LokiSource) StreamingAcquisition(ctx context.Context, out chan types.Event, t *tomb.Tomb) error { diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index d9c7c4301a4..24d73c4234c 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -443,13 +443,10 @@ func (s *S3Source) readFile(bucket string, key string) error { } else if s.MetricsLevel == configuration.METRICS_AGGREGATE { l.Src = bucket } - var evt types.Event - evt.Unmarshaled = make(map[string]interface{}) - if !s.Config.UseTimeMachine { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE} - } else { - evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE} - } + evt := types.MakeEvent(s.Config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG s.out <- evt } } diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 63d3e7a0239..5df1493ce13 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -235,11 +235,11 @@ func (s *SyslogSource) handleSyslogMsg(out chan types.Event, t *tomb.Tomb, c cha l.Time = ts l.Src = syslogLine.Client l.Process = true - if !s.config.UseTimeMachine { - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE, Unmarshaled: make(map[string]interface{})} - } else { - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} - } + evt := types.MakeEvent(s.config.UseTimeMachine) + evt.Line = l + evt.Process = true + evt.Type = types.LOG + out <- evt } } } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index a7da44b8524..53c2cee2493 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -430,7 +430,11 @@ OUTER_LOOP: l.Time = time.Now() l.Src = w.name l.Process = true - out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]interface{})} + csevt := types.MakeEvent(w.config.UseTimeMachine) + csevt.Line = l + csevt.Process = true + csevt.Type = types.LOG + out <- csevt } } } diff --git a/pkg/types/event.go b/pkg/types/event.go index e016d0294c4..a62c4258fd2 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -47,6 +47,19 @@ type Event struct { Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } +func MakeEvent(timeMachine bool) Event { + evt := Event{ + Parsed: make(map[string]string), + Meta: make(map[string]string), + Unmarshaled: make(map[string]interface{}), + ExpectMode: LIVE, + } + if timeMachine { + evt.ExpectMode = TIMEMACHINE + } + return evt +} + func (e *Event) SetMeta(key string, value string) bool { if e.Meta == nil { e.Meta = make(map[string]string) From 6d5df687c0735ebf77962209608b2b6f895d3907 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 28 Oct 2024 16:52:04 +0100 Subject: [PATCH 23/30] more parameters for MakeEvent --- pkg/acquisition/modules/appsec/utils.go | 5 +---- pkg/acquisition/modules/cloudwatch/cloudwatch.go | 4 +--- pkg/acquisition/modules/docker/docker.go | 6 ++---- pkg/acquisition/modules/file/file.go | 4 +--- pkg/acquisition/modules/http/http.go | 4 +--- pkg/acquisition/modules/journalctl/journalctl.go | 4 +--- pkg/acquisition/modules/kafka/kafka.go | 4 +--- pkg/acquisition/modules/kinesis/kinesis.go | 4 +--- pkg/acquisition/modules/kubernetesaudit/k8s_audit.go | 4 +--- pkg/acquisition/modules/loki/loki.go | 4 +--- pkg/acquisition/modules/s3/s3.go | 4 +--- pkg/acquisition/modules/syslog/syslog.go | 4 +--- pkg/acquisition/modules/wineventlog/wineventlog_windows.go | 4 +--- pkg/types/event.go | 4 +++- 14 files changed, 17 insertions(+), 42 deletions(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 4fb1a979d14..4f890dd513c 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -195,10 +195,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { } func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string) (types.Event, error) { - evt := types.Event{} - // we might want to change this based on in-band vs out-of-band ? - evt.Type = types.LOG - evt.ExpectMode = types.LIVE + evt := types.MakeEvent(false, types.LOG, true) // def needs fixing evt.Stage = "s00-raw" evt.Parsed = map[string]string{ diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index 3c027c41702..ba267c9050b 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -710,7 +710,7 @@ func (cw *CloudwatchSource) CatLogStream(ctx context.Context, cfg *LogStreamTail func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig) (types.Event, error) { l := types.Line{} - evt := types.MakeEvent(cfg.ExpectMode == types.TIMEMACHINE) + evt := types.MakeEvent(cfg.ExpectMode == types.TIMEMACHINE, types.LOG, true) if log.Message == nil { return evt, errors.New("nil message") } @@ -726,8 +726,6 @@ func cwLogToEvent(log *cloudwatchlogs.OutputLogEvent, cfg *LogStreamTailConfig) l.Process = true l.Module = "cloudwatch" evt.Line = l - evt.Process = true - evt.Type = types.LOG cfg.logger.Debugf("returned event labels : %+v", evt.Line.Labels) return evt, nil } diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index 2689d5ab9f0..b27255ec13f 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -334,7 +334,7 @@ func (d *DockerSource) OneShotAcquisition(ctx context.Context, out chan types.Ev if d.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"source": containerConfig.Name}).Inc() } - evt := types.MakeEvent(true) + evt := types.MakeEvent(true, types.LOG, true) evt.Line = l evt.Process = true evt.Type = types.LOG @@ -582,10 +582,8 @@ func (d *DockerSource) TailDocker(ctx context.Context, container *ContainerConfi l.Src = container.Name l.Process = true l.Module = d.GetName() - evt := types.MakeEvent(d.Config.UseTimeMachine) + evt := types.MakeEvent(d.Config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG linesRead.With(prometheus.Labels{"source": container.Name}).Inc() outChan <- evt d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw) diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 027bb323942..9f439b0c82e 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -621,10 +621,8 @@ func (f *FileSource) tailFile(out chan types.Event, t *tomb.Tomb, tail *tail.Tai // we're tailing, it must be real time logs logger.Debugf("pushing %+v", l) - evt := types.MakeEvent(f.config.UseTimeMachine) + evt := types.MakeEvent(f.config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG out <- evt } } diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 811396b5299..469188a8ce0 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -305,10 +305,8 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * line.Src = hc.Path } - evt := types.MakeEvent(h.Config.UseTimeMachine) + evt := types.MakeEvent(h.Config.UseTimeMachine, types.LOG, true) evt.Line = line - evt.Process = true - evt.Type = types.LOG if h.metricsLevel == configuration.METRICS_AGGREGATE { linesRead.With(prometheus.Labels{"path": hc.Path, "src": ""}).Inc() diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index ae4c32db81a..27f20b9f446 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -137,10 +137,8 @@ func (j *JournalCtlSource) runJournalCtl(ctx context.Context, out chan types.Eve linesRead.With(prometheus.Labels{"source": j.src}).Inc() } - evt := types.MakeEvent(j.config.UseTimeMachine) + evt := types.MakeEvent(j.config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG out <- evt case stderrLine := <-stderrChan: logger.Warnf("Got stderr message : %s", stderrLine) diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 6c44470601f..77fc44e310d 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -173,10 +173,8 @@ func (k *KafkaSource) ReadMessage(ctx context.Context, out chan types.Event) err if k.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"topic": k.Config.Topic}).Inc() } - evt := types.MakeEvent(k.Config.UseTimeMachine) + evt := types.MakeEvent(k.Config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG out <- evt } } diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index 632395c1eb2..3744e43f38d 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -322,10 +322,8 @@ func (k *KinesisSource) ParseAndPushRecords(records []*kinesis.Record, out chan } else { l.Src = k.Config.StreamName } - evt := types.MakeEvent(k.Config.UseTimeMachine) + evt := types.MakeEvent(k.Config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG out <- evt } } diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index 9272e6986cc..1fa6c894a32 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -207,10 +207,8 @@ func (ka *KubernetesAuditSource) webhookHandler(w http.ResponseWriter, r *http.R Process: true, Module: ka.GetName(), } - evt := types.MakeEvent(ka.config.UseTimeMachine) + evt := types.MakeEvent(ka.config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG ka.outChan <- evt } } diff --git a/pkg/acquisition/modules/loki/loki.go b/pkg/acquisition/modules/loki/loki.go index 88a9974ca07..d50787b652b 100644 --- a/pkg/acquisition/modules/loki/loki.go +++ b/pkg/acquisition/modules/loki/loki.go @@ -307,10 +307,8 @@ func (l *LokiSource) readOneEntry(entry lokiclient.Entry, labels map[string]stri if l.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"source": l.Config.URL}).Inc() } - evt := types.MakeEvent(l.Config.UseTimeMachine) + evt := types.MakeEvent(l.Config.UseTimeMachine, types.LOG, true) evt.Line = ll - evt.Process = true - evt.Type = types.LOG out <- evt } diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index 24d73c4234c..cdc84a8a3ca 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -443,10 +443,8 @@ func (s *S3Source) readFile(bucket string, key string) error { } else if s.MetricsLevel == configuration.METRICS_AGGREGATE { l.Src = bucket } - evt := types.MakeEvent(s.Config.UseTimeMachine) + evt := types.MakeEvent(s.Config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG s.out <- evt } } diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 5df1493ce13..fb6a04600c1 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -235,10 +235,8 @@ func (s *SyslogSource) handleSyslogMsg(out chan types.Event, t *tomb.Tomb, c cha l.Time = ts l.Src = syslogLine.Client l.Process = true - evt := types.MakeEvent(s.config.UseTimeMachine) + evt := types.MakeEvent(s.config.UseTimeMachine, types.LOG, true) evt.Line = l - evt.Process = true - evt.Type = types.LOG out <- evt } } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index 53c2cee2493..aa2a89a7754 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -430,10 +430,8 @@ OUTER_LOOP: l.Time = time.Now() l.Src = w.name l.Process = true - csevt := types.MakeEvent(w.config.UseTimeMachine) + csevt := types.MakeEvent(w.config.UseTimeMachine, types.LOG, true) csevt.Line = l - csevt.Process = true - csevt.Type = types.LOG out <- csevt } } diff --git a/pkg/types/event.go b/pkg/types/event.go index a62c4258fd2..a229ded333b 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -47,12 +47,14 @@ type Event struct { Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } -func MakeEvent(timeMachine bool) Event { +func MakeEvent(timeMachine bool, evtType int, process bool) Event { evt := Event{ Parsed: make(map[string]string), Meta: make(map[string]string), Unmarshaled: make(map[string]interface{}), ExpectMode: LIVE, + Process: process, + Type: evtType, } if timeMachine { evt.ExpectMode = TIMEMACHINE From bfd32a1eed50b21bbd757feb919426de3f181eb3 Mon Sep 17 00:00:00 2001 From: he2ss Date: Tue, 29 Oct 2024 11:38:36 +0100 Subject: [PATCH 24/30] fix laurence comments --- pkg/acquisition/modules/http/http.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 469188a8ce0..03d71e69b27 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -104,22 +104,23 @@ func (hc *HttpConfiguration) Validate() error { switch hc.AuthType { case "basic_auth": + baseErr := "basic_auth is selected, but" if hc.BasicAuth == nil { - return errors.New("basic_auth is required") + return errors.New(baseErr + " basic_auth is not provided") } if hc.BasicAuth.Username == "" { - return errors.New("username is required") + return errors.New(baseErr + " username is required") } if hc.BasicAuth.Password == "" { - return errors.New("password is required") + return errors.New(baseErr + " password is required") } case "headers": if hc.Headers == nil { - return errors.New("headers is required") + return errors.New("headers is selected, but headers is not provided") } case "mtls": if hc.TLS == nil || hc.TLS.CaCert == "" { - return errors.New("ca_cert is required") + return errors.New("mtls is selected, but ca_cert is not provided") } default: return errors.New("invalid auth_type: must be one of basic_auth, headers, mtls") From 85f74d00ae0ebe34ee6da509fe4b3c82c2e985a8 Mon Sep 17 00:00:00 2001 From: he2ss Date: Tue, 29 Oct 2024 11:41:49 +0100 Subject: [PATCH 25/30] fix laurence comments --- pkg/acquisition/modules/http/http.go | 4 ++-- pkg/acquisition/modules/http/http_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 03d71e69b27..98af134c84e 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -109,10 +109,10 @@ func (hc *HttpConfiguration) Validate() error { return errors.New(baseErr + " basic_auth is not provided") } if hc.BasicAuth.Username == "" { - return errors.New(baseErr + " username is required") + return errors.New(baseErr + " username is not provided") } if hc.BasicAuth.Password == "" { - return errors.New(baseErr + " password is required") + return errors.New(baseErr + " password is not provided") } case "headers": if hc.Headers == nil { diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index aac7a5b7869..d79ef5d3869 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -49,7 +49,7 @@ source: http listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth`, - expectedErr: "invalid configuration: basic_auth is required", + expectedErr: "invalid configuration: basic_auth is selected, but basic_auth is not provided", }, { config: ` @@ -57,7 +57,7 @@ source: http listen_addr: 127.0.0.1:8080 path: /test auth_type: headers`, - expectedErr: "invalid configuration: headers is required", + expectedErr: "invalid configuration: headers is selected, but headers is not provided", }, { config: ` @@ -67,7 +67,7 @@ path: /test auth_type: basic_auth basic_auth: username: 132`, - expectedErr: "invalid configuration: password is required", + expectedErr: "invalid configuration: basic_auth is selected, but password is not provided", }, { config: ` @@ -77,7 +77,7 @@ path: /test auth_type: basic_auth basic_auth: password: 132`, - expectedErr: "invalid configuration: username is required", + expectedErr: "invalid configuration: basic_auth is selected, but username is not provided", }, { config: ` @@ -86,7 +86,7 @@ listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers:`, - expectedErr: "invalid configuration: headers is required", + expectedErr: "invalid configuration: headers is selected, but headers is not provided", }, { config: ` @@ -129,7 +129,7 @@ auth_type: mtls tls: server_cert: cert server_key: key`, - expectedErr: "invalid configuration: ca_cert is required", + expectedErr: "invalid configuration: mtls is selected, but ca_cert is not provided", }, { config: ` From 99a988e82d013a5c648241bef255277262416f53 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 29 Oct 2024 16:26:30 +0100 Subject: [PATCH 26/30] properly copy event in transform --- pkg/acquisition/acquisition.go | 24 +++++++++++++++++------- pkg/types/event.go | 1 + 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 1ad385105d3..ef5a413b91f 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -337,6 +337,20 @@ func GetMetrics(sources []DataSource, aggregated bool) error { return nil } +// There's no need for an actual deep copy +// The event is almost empty, we are mostly interested in allocating new maps for Parsed/Meta/... +func copyEvent(evt types.Event, line string) types.Event { + evtCopy := types.MakeEvent(evt.ExpectMode == types.TIMEMACHINE, evt.Type, evt.Process) + evtCopy.Line = evt.Line + evtCopy.Line.Raw = line + evtCopy.Line.Labels = make(map[string]string) + for k, v := range evt.Line.Labels { + evtCopy.Line.Labels[k] = v + } + + return evtCopy +} + func transform(transformChan chan types.Event, output chan types.Event, AcquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) { defer trace.CatchPanic("crowdsec/acquis") logger.Infof("transformer started") @@ -363,8 +377,7 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo switch v := out.(type) { case string: logger.Tracef("transform expression returned %s", v) - evt.Line.Raw = v - output <- evt + output <- copyEvent(evt, v) case []interface{}: logger.Tracef("transform expression returned %v", v) //nolint:asasalint // We actually want to log the slice content @@ -373,19 +386,16 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo if !ok { logger.Errorf("transform expression returned []interface{}, but cannot assert an element to string") output <- evt - continue } - evt.Line.Raw = l - output <- evt + output <- copyEvent(evt, l) } case []string: logger.Tracef("transform expression returned %v", v) for _, line := range v { - evt.Line.Raw = line - output <- evt + output <- copyEvent(evt, line) } default: logger.Errorf("transform expression returned an invalid type %T, sending event as-is", out) diff --git a/pkg/types/event.go b/pkg/types/event.go index a229ded333b..9300626b927 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -52,6 +52,7 @@ func MakeEvent(timeMachine bool, evtType int, process bool) Event { Parsed: make(map[string]string), Meta: make(map[string]string), Unmarshaled: make(map[string]interface{}), + Enriched: make(map[string]string), ExpectMode: LIVE, Process: process, Type: evtType, From a1712743e4697bd5f1ebd78a8d83ef91ce4b34ee Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 1 Nov 2024 09:33:29 +0100 Subject: [PATCH 27/30] list build tag in makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 87bb0313b25..4b7f0b746fe 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,7 @@ COMPONENTS := \ datasource_cloudwatch \ datasource_docker \ datasource_file \ + datasource_http \ datasource_k8saudit \ datasource_kafka \ datasource_journalctl \ From 8605cec663acb2e69fefd5bca7648ebc4ae9c14b Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 4 Nov 2024 10:51:10 +0100 Subject: [PATCH 28/30] test: use assertions --- pkg/acquisition/modules/http/http_test.go | 274 +++++++--------------- 1 file changed, 86 insertions(+), 188 deletions(-) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index d79ef5d3869..f1c90941ed8 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -19,6 +19,8 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( @@ -180,9 +182,7 @@ custom_status_code: 999`, func TestGetUuid(t *testing.T) { h := HTTPSource{} h.Config.UniqueId = "test" - if h.GetUuid() != "test" { - t.Fatalf("expected 'test', got '%s'", h.GetUuid()) - } + assert.Equal(t, "test", h.GetUuid()) } func TestUnmarshalConfig(t *testing.T) { @@ -210,16 +210,12 @@ func TestConfigureByDSN(t *testing.T) { func TestGetMode(t *testing.T) { h := HTTPSource{} h.Config.Mode = "test" - if h.GetMode() != "test" { - t.Fatalf("expected 'test', got '%s'", h.GetMode()) - } + assert.Equal(t, "test", h.GetMode()) } func TestGetName(t *testing.T) { h := HTTPSource{} - if h.GetName() != "http" { - t.Fatalf("expected 'http', got '%s'", h.GetName()) - } + assert.Equal(t, "http", h.GetName()) } func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLevel int) (chan types.Event, *tomb.Tomb) { @@ -228,15 +224,11 @@ func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLev "type": "http", }) err := h.Configure(config, subLogger, metricLevel) - if err != nil { - t.Fatalf("unable to configure http source: %s", err) - } + require.NoError(t, err) tomb := tomb.Tomb{} out := make(chan types.Event) err = h.StreamingAcquisition(ctx, out, &tomb) - if err != nil { - t.Fatalf("unable to start streaming acquisition: %s", err) - } + require.NoError(t, err) for _, metric := range h.GetMetrics() { prometheus.Register(metric) @@ -259,12 +251,8 @@ basic_auth: time.Sleep(1 * time.Second) res, err := http.Get(fmt.Sprintf("%s/test", testHTTPServerAddr)) - if err != nil { - t.Fatalf("unable to get http response: %s", err) - } - if res.StatusCode != http.StatusMethodNotAllowed { - t.Fatalf("expected status code %d, got %d", http.StatusMethodNotAllowed, res.StatusCode) - } + require.NoError(t, err) + assert.Equal(t, http.StatusMethodNotAllowed, res.StatusCode) h.Server.Close() tomb.Kill(nil) @@ -286,13 +274,8 @@ basic_auth: time.Sleep(1 * time.Second) res, err := http.Get(fmt.Sprintf("%s/unknown", testHTTPServerAddr)) - if err != nil { - t.Fatalf("unable to get http response: %s", err) - } - - if res.StatusCode != http.StatusNotFound { - t.Fatalf("expected status code %d, got %d", http.StatusNotFound, res.StatusCode) - } + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, res.StatusCode) h.Server.Close() tomb.Kill(nil) @@ -315,25 +298,16 @@ basic_auth: client := &http.Client{} resp, err := http.Post(fmt.Sprintf("%s/test", testHTTPServerAddr), "application/json", strings.NewReader("test")) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusUnauthorized { - t.Fatalf("expected status code %d, got %d", http.StatusUnauthorized, resp.StatusCode) - } + require.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader("test")) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) req.SetBasicAuth("test", "WrongPassword") + resp, err = client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusUnauthorized { - t.Fatalf("expected status code %d, got %d", http.StatusUnauthorized, resp.StatusCode) - } + require.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) h.Server.Close() tomb.Kill(nil) @@ -355,17 +329,12 @@ headers: client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader("test")) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) + req.Header.Add("Key", "wrong") resp, err := client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusUnauthorized { - t.Fatalf("expected status code %d, got %d", http.StatusUnauthorized, resp.StatusCode) - } + require.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) h.Server.Close() tomb.Kill(nil) @@ -387,17 +356,13 @@ max_body_size: 5`), 0) client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader("testtest")) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) + req.Header.Add("Key", "test") resp, err := client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusRequestEntityTooLarge { - t.Fatalf("expected status code %d, got %d", http.StatusRequestEntityTooLarge, resp.StatusCode) - } + require.NoError(t, err) + + assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode) h.Server.Close() tomb.Kill(nil) @@ -422,22 +387,15 @@ headers: client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) + req.Header.Add("Key", "test") resp, err := client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) - } + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) err = <-errChan - if err != nil { - t.Fatalf("error: %s", err) - } + require.NoError(t, err) assertMetrics(t, h.GetMetrics(), 1) @@ -466,27 +424,17 @@ custom_headers: go assertEvents(out, []string{rawEvt}, errChan) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(rawEvt)) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) + req.Header.Add("Key", "test") resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } + require.NoError(t, err) - if resp.StatusCode != http.StatusCreated { - t.Fatalf("expected status code %d, got %d", http.StatusCreated, resp.StatusCode) - } - - if resp.Header.Get("Success") != "true" { - t.Fatalf("expected header 'success' to be 'true', got '%s'", resp.Header.Get("Success")) - } + assert.Equal(t, http.StatusCreated, resp.StatusCode) + assert.Equal(t, "true", resp.Header.Get("Success")) err = <-errChan - if err != nil { - t.Fatalf("error: %s", err) - } + require.NoError(t, err) assertMetrics(t, h.GetMetrics(), 1) @@ -565,20 +513,16 @@ timeout: 1s`), 0) } req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), slow) - if err != nil { - t.Fatalf("Error creating request: %v", err) - } + require.NoError(t, err) + req.Header.Add("Key", "test") req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) - if err != nil { - t.Fatalf("Error sending request: %v", err) - } - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("expected status code %d, got %d", http.StatusBadRequest, resp.StatusCode) - } + require.NoError(t, err) + + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) h.Server.Close() tomb.Kill(nil) @@ -600,13 +544,9 @@ tls: time.Sleep(1 * time.Second) resp, err := http.Post(fmt.Sprintf("%s/test", testHTTPServerAddr), "application/json", strings.NewReader("test")) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } + require.NoError(t, err) - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("expected status code %d, got %d", http.StatusBadRequest, resp.StatusCode) - } + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) h.Server.Close() tomb.Kill(nil) @@ -630,9 +570,8 @@ tls: time.Sleep(1 * time.Second) caCert, err := os.ReadFile("testdata/server.crt") - if err != nil { - t.Fatalf("unable to read ca cert: %s", err) - } + require.NoError(t, err) + caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) @@ -651,22 +590,16 @@ tls: go assertEvents(out, []string{rawEvt}, errChan) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) + req.Header.Add("Key", "test") resp, err := client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) - } + require.NoError(t, err) + + assert.Equal(t, http.StatusOK, resp.StatusCode) err = <-errChan - if err != nil { - t.Fatalf("error: %s", err) - } + require.NoError(t, err) assertMetrics(t, h.GetMetrics(), 0) @@ -691,14 +624,10 @@ tls: // init client cert cert, err := tls.LoadX509KeyPair("testdata/client.crt", "testdata/client.key") - if err != nil { - t.Fatalf("unable to load client cert: %s", err) - } + require.NoError(t, err) caCert, err := os.ReadFile("testdata/ca.crt") - if err != nil { - t.Fatalf("unable to read ca cert: %s", err) - } + require.NoError(t, err) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) @@ -719,23 +648,15 @@ tls: go assertEvents(out, []string{rawEvt}, errChan) req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddrTLS), strings.NewReader(rawEvt)) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) resp, err := client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } + require.NoError(t, err) - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) - } + assert.Equal(t, http.StatusOK, resp.StatusCode) err = <-errChan - if err != nil { - t.Fatalf("error: %s", err) - } + require.NoError(t, err) assertMetrics(t, h.GetMetrics(), 0) @@ -762,38 +683,32 @@ headers: var b strings.Builder gz := gzip.NewWriter(&b) - if _, err := gz.Write([]byte(rawEvt)); err != nil { - t.Fatalf("unable to write gzipped data: %s", err) - } - if _, err := gz.Write([]byte(rawEvt)); err != nil { - t.Fatalf("unable to write gzipped data: %s", err) - } - if err := gz.Close(); err != nil { - t.Fatalf("unable to close gzip writer: %s", err) - } + + _, err := gz.Write([]byte(rawEvt)) + require.NoError(t, err) + + _, err = gz.Write([]byte(rawEvt)) + require.NoError(t, err) + + err = gz.Close() + require.NoError(t, err) // send gzipped compressed data client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(b.String())) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) + req.Header.Add("Key", "test") req.Header.Add("Content-Encoding", "gzip") req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) - } + require.NoError(t, err) + + assert.Equal(t, http.StatusOK, resp.StatusCode) err = <-errChan - if err != nil { - t.Fatalf("error: %s", err) - } + require.NoError(t, err) assertMetrics(t, h.GetMetrics(), 2) @@ -821,25 +736,17 @@ headers: client := &http.Client{} req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/test", testHTTPServerAddr), strings.NewReader(fmt.Sprintf("%s\n%s\n", rawEvt, rawEvt))) - if err != nil { - t.Fatalf("unable to create http request: %s", err) - } + require.NoError(t, err) req.Header.Add("Key", "test") req.Header.Add("Content-Type", "application/x-ndjson") resp, err := client.Do(req) - if err != nil { - t.Fatalf("unable to post http request: %s", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected status code %d, got %d", http.StatusOK, resp.StatusCode) - } + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) err = <-errChan - if err != nil { - t.Fatalf("error: %s", err) - } + require.NoError(t, err) assertMetrics(t, h.GetMetrics(), 2) @@ -850,30 +757,21 @@ headers: func assertMetrics(t *testing.T, metrics []prometheus.Collector, expected int) { promMetrics, err := prometheus.DefaultGatherer.Gather() - if err != nil { - t.Fatalf("unable to gather metrics: %s", err) - } + require.NoError(t, err) + isExist := false for _, metricFamily := range promMetrics { if metricFamily.GetName() == "cs_httpsource_hits_total" { isExist = true - if len(metricFamily.GetMetric()) != 1 { - t.Fatalf("expected 1 metricFamily, got %d", len(metricFamily.GetMetric())) - } + assert.Len(t, metricFamily.GetMetric(), 1) for _, metric := range metricFamily.GetMetric() { - if metric.GetCounter().GetValue() != float64(expected) { - t.Fatalf("expected %d, got %f", expected, metric.GetCounter().GetValue()) - } + assert.Equal(t, float64(expected), metric.GetCounter().GetValue()) labels := metric.GetLabel() - if len(labels) != 2 { - t.Fatalf("expected 2 label, got %d", len(labels)) - } - if labels[0].GetName() != "path" || labels[0].GetValue() != "/test" { - t.Fatalf("expected label path:/test, got %s:%s", labels[0].GetName(), labels[0].GetValue()) - } - if labels[1].GetName() != "src" || labels[1].GetValue() != "127.0.0.1" { - t.Fatalf("expected label src:127.0.0.1, got %s:%s", labels[1].GetName(), labels[1].GetValue()) - } + assert.Len(t, labels, 2) + assert.Equal(t, "path", labels[0].GetName()) + assert.Equal(t, "/test", labels[0].GetValue()) + assert.Equal(t, "src", labels[1].GetName()) + assert.Equal(t, "127.0.0.1", labels[1].GetValue()) } } } From 2293770856aab262515e1d5aa72ad2ea1cc4adcc Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 4 Nov 2024 11:59:32 +0100 Subject: [PATCH 29/30] types.NewEvent() -> types.MakeEvent() --- pkg/alertcontext/alertcontext.go | 10 ++++------ pkg/types/event.go | 9 --------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/pkg/alertcontext/alertcontext.go b/pkg/alertcontext/alertcontext.go index 0c60dea4292..1b7d1e20018 100644 --- a/pkg/alertcontext/alertcontext.go +++ b/pkg/alertcontext/alertcontext.go @@ -149,15 +149,12 @@ func TruncateContext(values []string, contextValueLen int) (string, error) { return ret, nil } -func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, request *http.Request, tmpContext map[string][]string) []error { +func EvalAlertContextRules(evt types.Event, match *types.MatchedRule, request *http.Request, tmpContext map[string][]string) []error { var errors []error //if we're evaluating context for appsec event, match and request will be present. //otherwise, only evt will be. - if evt == nil { - evt = types.NewEvent() - } if match == nil { match = types.NewMatchedRule() } @@ -220,8 +217,9 @@ func AppsecEventToContext(event types.AppsecEvent, request *http.Request) (model tmpContext := make(map[string][]string) + evt := types.MakeEvent(false, types.LOG, false) for _, matched_rule := range event.MatchedRules { - tmpErrors := EvalAlertContextRules(nil, &matched_rule, request, tmpContext) + tmpErrors := EvalAlertContextRules(evt, &matched_rule, request, tmpContext) errors = append(errors, tmpErrors...) } @@ -240,7 +238,7 @@ func EventToContext(events []types.Event) (models.Meta, []error) { tmpContext := make(map[string][]string) for _, evt := range events { - tmpErrors := EvalAlertContextRules(&evt, nil, nil, tmpContext) + tmpErrors := EvalAlertContextRules(evt, nil, nil, tmpContext) errors = append(errors, tmpErrors...) } diff --git a/pkg/types/event.go b/pkg/types/event.go index f2698178cac..9300626b927 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -63,15 +63,6 @@ func MakeEvent(timeMachine bool, evtType int, process bool) Event { return evt } -func NewEvent() *Event { - return &Event{Type: LOG, - Parsed: make(map[string]string), - Enriched: make(map[string]string), - Meta: make(map[string]string), - Unmarshaled: make(map[string]interface{}), - } -} - func (e *Event) SetMeta(key string, value string) bool { if e.Meta == nil { e.Meta = make(map[string]string) From aadc31afb93cb8851dfd0dcce53b99d419ef7c23 Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 4 Nov 2024 12:18:43 +0100 Subject: [PATCH 30/30] float comparison with assert.InDelta() --- pkg/acquisition/modules/http/http_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index f1c90941ed8..f89ba7aa8ba 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -765,7 +765,7 @@ func assertMetrics(t *testing.T, metrics []prometheus.Collector, expected int) { isExist = true assert.Len(t, metricFamily.GetMetric(), 1) for _, metric := range metricFamily.GetMetric() { - assert.Equal(t, float64(expected), metric.GetCounter().GetValue()) + assert.InDelta(t, float64(expected), metric.GetCounter().GetValue(), 0.000001) labels := metric.GetLabel() assert.Len(t, labels, 2) assert.Equal(t, "path", labels[0].GetName())