diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 2ac4a8911a1..f5ca39b3645 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -27,6 +27,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "Auditbeat Check/Update" + key: "auditbeat-check-update" + + steps: + - label: "Run check/update" + command: | + make -C auditbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "auditbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "auditbeat-check-update" + - group: "Auditbeat Mandatory Testing" key: "auditbeat-mandatory-tests" diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh index 7e79d2d3164..1ab80edc585 100644 --- a/.buildkite/scripts/changesets.sh +++ b/.buildkite/scripts/changesets.sh @@ -15,7 +15,7 @@ definePattern() { } defineExclusions() { - exclude="^$beatPath\/module\/(.*(? Replicaset -> Pod // CronJob -> job -> Pod if metaConf.Deployment { replicaSetWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_rs", client, &kubernetes.ReplicaSet{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.ReplicaSet{}, err) @@ -133,7 +137,9 @@ func NewPodEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publish fu } if metaConf.CronJob { jobWatcher, err = kubernetes.NewNamedWatcher("resource_metadata_enricher_job", client, &kubernetes.Job{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { logger.Errorf("Error creating watcher for %T due to error %+v", &kubernetes.Job{}, err) diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index ba62dda9c47..e9e71c921bd 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -74,8 +74,9 @@ func NewServiceEventer(uuid uuid.UUID, cfg *conf.C, client k8s.Interface, publis if metaConf.Namespace.Enabled() || config.Hints.Enabled() { namespaceWatcher, err = kubernetes.NewNamedWatcher("namespace", client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ - SyncTimeout: config.SyncPeriod, - Namespace: config.Namespace, + SyncTimeout: config.SyncPeriod, + Namespace: config.Namespace, + HonorReSyncs: true, }, nil) if err != nil { return nil, fmt.Errorf("couldn't create watcher for %T due to error %w", &kubernetes.Namespace{}, err) diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index bad3f87b38d..3dc94738d12 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> diff --git a/libbeat/docs/shared-autodiscover.asciidoc b/libbeat/docs/shared-autodiscover.asciidoc index 1e1ea567b7c..83d44b498ac 100644 --- a/libbeat/docs/shared-autodiscover.asciidoc +++ b/libbeat/docs/shared-autodiscover.asciidoc @@ -140,10 +140,7 @@ The `kubernetes` autodiscover provider has the following configuration settings: `node`:: (Optional) Specify the node to scope {beatname_lc} to in case it cannot be accurately detected, as when running {beatname_lc} in host network mode. -`namespace`:: (Optional) Select the namespace from which to collect the - metadata. If it is not set, the processor collects metadata from all - namespaces. It is unset by default. The namespace configuration only applies to - kubernetes resources that are namespace scoped. +`namespace`:: (Optional) Select the namespace from which to collect the events from the resources. If it is not set, the provider collects them from all namespaces. It is unset by default. The namespace configuration only applies to kubernetes resources that are namespace scoped and if `unique` field is set to `false`. `cleanup_timeout`:: (Optional) Specify the time of inactivity before stopping the running configuration for a container, ifeval::["{beatname_lc}"=="filebeat"] @@ -196,7 +193,7 @@ Example: `unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into making the provider to enable the provided templates only when it will gain the leader lease. - This setting can only be combined with `cluster` scope. When `unique` is enabled enabled, `resource` + This setting can only be combined with `cluster` scope. When `unique` is enabled, `resource` and `add_resource_metadata` settings are not taken into account. `leader_lease`:: (Optional) Defaults to +{beatname_lc}-cluster-leader+. This will be name of the lock lease. One can monitor the status of the lease with `kubectl describe lease beats-cluster-leader`. diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index cb37384bb30..03efb1aa419 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.15.0 :doc-branch: main -:go-version: 1.21.10 +:go-version: 1.21.11 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index ce43c81c312..de48ddeff9d 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ RUN \ diff --git a/metricbeat/docs/running-on-docker.asciidoc b/metricbeat/docs/running-on-docker.asciidoc index 72cfed9757d..26ba27658b8 100644 --- a/metricbeat/docs/running-on-docker.asciidoc +++ b/metricbeat/docs/running-on-docker.asciidoc @@ -21,6 +21,7 @@ docker run \ --mount type=bind,source=/var/run/dbus/system_bus_socket,target=/hostfs/var/run/dbus/system_bus_socket,readonly \ <4> --env DBUS_SYSTEM_BUS_ADDRESS='unix:path=/hostfs/var/run/dbus/system_bus_socket' \ <4> --net=host \ <5> + --cgroupns=host \ <6> {dockerimage} -e -system.hostfs=/hostfs ---- @@ -45,6 +46,7 @@ both require access to dbus. Mount the dbus socket and set the `DBUS_SYSTEM_BUS_ to make this file contain the host's network devices is to use the `--net=host` flag. This is due to Linux namespacing; simply bind mounting the host's `/proc` to `/hostfs/proc` is not sufficient. +<6> Runs the container using the host's cgroup namespace, instead of a private namespace. While this is optional, <> may produce more correct cgroup metrics when running in host mode. NOTE: The special filesystems +/proc+ and +/sys+ are only available if the host system is running Linux. Attempts to bind-mount these filesystems will diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 0a5646a9d75..b6517fea093 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 COPY test/main.go main.go diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index f3cab807dfd..f443a51dd3a 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.21.10 AS build-env +FROM golang:1.21.11 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index 9dea6777c57..6301d44f4e2 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.21.10 +FROM golang:1.21.11 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 912cd87cc43..22c50eeb430 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 RUN \ apt-get update \ diff --git a/x-pack/filebeat/input/http_endpoint/handler.go b/x-pack/filebeat/input/http_endpoint/handler.go index b799248a935..4f3fdd550aa 100644 --- a/x-pack/filebeat/input/http_endpoint/handler.go +++ b/x-pack/filebeat/input/http_endpoint/handler.go @@ -18,13 +18,13 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "time" "github.com/google/cel-go/cel" "github.com/google/cel-go/checker/decls" "github.com/google/cel-go/common/types" "github.com/google/cel-go/common/types/ref" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/structpb" @@ -53,8 +53,8 @@ type handler struct { publish func(beat.Event) log *logp.Logger validator apiValidator - txBaseID string // Random value to make transaction IDs unique. - txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter atomic.Uint64 // Transaction ID counter that is incremented for each request. reqLogger *zap.Logger host, scheme string @@ -290,7 +290,7 @@ func (h *handler) logRequest(txID string, r *http.Request, status int, respBody } func (h *handler) nextTxID() string { - count := h.txIDCounter.Inc() + count := h.txIDCounter.Add(1) return h.formatTxID(count) } diff --git a/x-pack/filebeat/input/http_endpoint/input.go b/x-pack/filebeat/input/http_endpoint/input.go index 6737a9b9aa0..7f0440deb60 100644 --- a/x-pack/filebeat/input/http_endpoint/input.go +++ b/x-pack/filebeat/input/http_endpoint/input.go @@ -24,7 +24,6 @@ import ( "github.com/rcrowley/go-metrics" "go.elastic.co/ecszap" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -327,10 +326,9 @@ func (s *server) getErr() error { func newHandler(ctx context.Context, c config, prg *program, pub func(beat.Event), log *logp.Logger, metrics *inputMetrics) http.Handler { h := &handler{ - ctx: ctx, - log: log, - txBaseID: newID(), - txIDCounter: atomic.NewUint64(0), + ctx: ctx, + log: log, + txBaseID: newID(), publish: pub, metrics: metrics, diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index ce68147a2a7..9e60cb60942 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -15,9 +15,9 @@ import ( "net/http" "net/http/httputil" "strconv" + "sync/atomic" "time" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -36,22 +36,21 @@ type contextKey string // responses to the provided logger. Transaction creation is logged to log. func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger, maxBodyLen int, log *logp.Logger) *LoggingRoundTripper { return &LoggingRoundTripper{ - transport: next, - maxBodyLen: maxBodyLen, - txLog: logger, - txBaseID: newID(), - txIDCounter: atomic.NewUint64(0), - log: log, + transport: next, + maxBodyLen: maxBodyLen, + txLog: logger, + txBaseID: newID(), + log: log, } } // LoggingRoundTripper is an http.RoundTripper that logs requests and responses. type LoggingRoundTripper struct { transport http.RoundTripper - maxBodyLen int // The maximum length of a body. Longer bodies will be truncated. - txLog *zap.Logger // Destination logger. - txBaseID string // Random value to make transaction IDs unique. - txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. + maxBodyLen int // The maximum length of a body. Longer bodies will be truncated. + txLog *zap.Logger // Destination logger. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter atomic.Uint64 // Transaction ID counter that is incremented for each request. log *logp.Logger } @@ -220,7 +219,7 @@ func (rt *LoggingRoundTripper) TxID() string { // nextTxID returns the next transaction.id value. It increments the internal // request counter. func (rt *LoggingRoundTripper) nextTxID() string { - count := rt.txIDCounter.Inc() + count := rt.txIDCounter.Add(1) return rt.formatTxID(count) } diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 3b3a785a0ca..d1c4d5af3cd 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.10 +FROM golang:1.21.11 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go index fe7473d0f3d..3679f07fb54 100644 --- a/x-pack/functionbeat/manager/aws/event_stack_poller_test.go +++ b/x-pack/functionbeat/manager/aws/event_stack_poller_test.go @@ -7,6 +7,7 @@ package aws import ( "context" "strconv" + "sync/atomic" "testing" "time" @@ -14,7 +15,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" "github.com/elastic/elastic-agent-libs/logp" ) @@ -29,7 +29,7 @@ func (m *mockEventHandler) sync(event types.StackEvent) bool { if m.skipCount.Load() >= m.skipEvents { return false } - m.skipCount.Inc() + m.skipCount.Add(1) return true } diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json index f093d13a3e4..216971d16b5 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json @@ -46,7 +46,7 @@ }, "panelIndex": "2", "panelRefName": "panel_2", - "title": "Tablespace Total Size", + "title": "Tablespace Max Total Size", "type": "visualization", "version": "8.0.0-SNAPSHOT" }, diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json index 603fcc02ec2..27de5af579a 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json @@ -4,7 +4,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": {} }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "uiStateJSON": {}, "version": 1, "visState": { @@ -24,13 +24,40 @@ "fill": 0.5, "formatter": "bytes", "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Tablespace total size", + "label": "Tablespace max total size", "line_width": 1, "metrics": [ { - "field": "oracle.tablespace.space.total.bytes", + "agg_with": "avg", + "field": "oracle.tablespace.space.used.bytes", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" + }, + { + "agg_with": "avg", + "colors": [ + "#68BC00" + ], + "field": "oracle.tablespace.space.free.bytes", + "id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "type": "max" + }, + { + "id": "2cf57800-8b54-41fa-a877-159b49699a50", + "script": "params.used_bytes + params.free_bytes", + "type": "math", + "variables": [ + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "631a44d5-d18a-4743-bea0-6f61930fd65f", + "name": "used_bytes" + }, + { + "field": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "id": "c255d24c-3a29-4879-b999-77af43d97c6b", + "name": "free_bytes" + } + ] } ], "point_size": 1, @@ -48,7 +75,7 @@ "type": "timeseries", "use_kibana_indexes": false }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "type": "metrics" } }, diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index 30c86d766da..82f93a060b1 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -16,8 +16,9 @@ import ( // extract is the E of a ETL processing. Gets the data files, used/free space and temp free space data that is fetch // by doing queries to Oracle -func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (out *extractedData, err error) { - out = &extractedData{} +func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (*extractedData, error) { + out := &extractedData{} + var err error if out.dataFiles, err = extractor.dataFilesData(ctx); err != nil { return nil, fmt.Errorf("error getting data_files: %w", err) @@ -31,23 +32,23 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth return nil, fmt.Errorf("error getting free space data: %w", err) } - return + return out, nil } // transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data // to create a Kibana/Elasticsearch friendly JSON. Data from Oracle is pretty fragmented by design so a lot of data // was necessary. Data is organized by Tablespace entity (Tablespaces might contain one or more data files) -func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { - out = make(map[string]mapstr.M, 0) +func (m *MetricSet) transform(in *extractedData) map[string]mapstr.M { + out := make(map[string]mapstr.M, 0) - for _, dataFile := range in.dataFiles { - m.addDataFileData(&dataFile, out) + for i := range in.dataFiles { + m.addDataFileData(&in.dataFiles[i], out) } m.addUsedAndFreeSpaceData(in.freeSpace, out) m.addTempFreeSpaceData(in.tempFreeSpace, out) - return + return out } func (m *MetricSet) extractAndTransform(ctx context.Context) ([]mb.Event, error) { @@ -78,7 +79,7 @@ func (m *MetricSet) addTempFreeSpaceData(tempFreeSpaces []tempFreeSpace, out map name := val.(string) if name == "TEMP" { for _, tempFreeSpaceTable := range tempFreeSpaces { - oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TablespaceSize}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TotalSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.UsedSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.FreeSpace}) } @@ -101,6 +102,7 @@ func (m *MetricSet) addUsedAndFreeSpaceData(freeSpaces []usedAndFreeSpace, out m if name == freeSpaceTable.TablespaceName { oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalFreeBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalUsedBytes}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalSpaceBytes}) } } } diff --git a/x-pack/metricbeat/module/oracle/tablespace/data_test.go b/x-pack/metricbeat/module/oracle/tablespace/data_test.go index 02e4bdd4528..9c6990a2598 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data_test.go @@ -12,13 +12,13 @@ import ( "github.com/stretchr/testify/assert" ) -var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"used":{"bytes":9991}}}`, +var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"total":{"bytes":99999},"used":{"bytes":9991}}}`, `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/temp012017-03-02_07-54-38-075-AM.dbf","online_status":"ONLINE","size":{"bytes":999991,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"TEMP","space":{"free":{"bytes":99999},"total":{"bytes":99999},"used":{"bytes":99999}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`} + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`} var notExpectedEvents = []string{`{}`, `{"foo":"bar"}`} diff --git a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go index 12348236bcc..9f5800166af 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go @@ -77,16 +77,16 @@ func (h happyDataFiles) dataFilesData(_ context.Context) ([]dataFile, error) { type happyTempFreeSpaceData struct{} func (happyTempFreeSpaceData) tempFreeSpaceData(_ context.Context) ([]tempFreeSpace, error) { - return []tempFreeSpace{{TablespaceName: "TEMP", TablespaceSize: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil + return []tempFreeSpace{{TablespaceName: "TEMP", TotalSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil } type happyFreeSpaceData struct{} func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) { return []usedAndFreeSpace{ - {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, }, nil } diff --git a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go index efc8401e2c0..f4b592d2b75 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go @@ -11,31 +11,24 @@ import ( ) type tempFreeSpace struct { - TablespaceName string - TablespaceSize sql.NullInt64 - UsedSpaceBytes sql.NullInt64 - FreeSpace sql.NullInt64 -} - -func (d *tempFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *tempFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + UsedSpaceBytes sql.NullInt64 + FreeSpace sql.NullInt64 } func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT TABLESPACE_NAME, TABLESPACE_SIZE, ALLOCATED_SPACE, FREE_SPACE FROM DBA_TEMP_FREE_SPACE") + rows, err := e.db.QueryContext(ctx, `SELECT t.TABLESPACE_NAME, (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t `) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]tempFreeSpace, 0) for rows.Next() { dest := tempFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TablespaceSize, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalSpaceBytes, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { return nil, err } results = append(results, dest) diff --git a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go index b17b249808d..e327badcf02 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go @@ -11,30 +11,24 @@ import ( ) type usedAndFreeSpace struct { - TablespaceName string - TotalFreeBytes sql.NullInt64 - TotalUsedBytes sql.NullInt64 -} - -func (d *usedAndFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *usedAndFreeSpace) eventKey() string { - return d.TablespaceName + TablespaceName string + TotalSpaceBytes sql.NullInt64 + TotalFreeBytes sql.NullInt64 + TotalUsedBytes sql.NullInt64 } func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT b.tablespace_name, tbs_size used, a.free_space free FROM (SELECT tablespace_name, sum(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, sum(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+)=b.tablespace_name") + rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - NVL(a.free_space, 0)) AS used, NVL(a.free_space, 0) AS free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]usedAndFreeSpace, 0) for rows.Next() { dest := usedAndFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes, &dest.TotalSpaceBytes); err != nil { return nil, err } results = append(results, dest) diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index 5023acb7b46..e7fd7774af8 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.21.10 AS build-env +FROM golang:1.21.11 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build .