diff --git a/NOTICE.txt b/NOTICE.txt index 3f70be40746..5c1ed0b8277 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -288,11 +288,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-data -Version: v0.1.1-0.20230718152028-9c38d2361527 +Version: v0.1.1-0.20230727133025-e3e170b1fded Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20230718152028-9c38d2361527/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v0.1.1-0.20230727133025-e3e170b1fded/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 47416178b45..0a3b700efa0 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b github.com/dustin/go-humanize v1.0.1 - github.com/elastic/apm-data v0.1.1-0.20230718152028-9c38d2361527 + github.com/elastic/apm-data v0.1.1-0.20230727133025-e3e170b1fded github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230727143857-7fd27bf1f6c4 github.com/elastic/elastic-agent-client/v7 v7.1.2 github.com/elastic/elastic-agent-libs v0.3.9 diff --git a/go.sum b/go.sum index a6c3d039db4..254f91c0bea 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9 github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/apm-data v0.1.1-0.20230718152028-9c38d2361527 h1:ASiUA/r0Wwnhn3HXvNuLzn23ljsKJeQRZsDdODlxMco= -github.com/elastic/apm-data v0.1.1-0.20230718152028-9c38d2361527/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= +github.com/elastic/apm-data v0.1.1-0.20230727133025-e3e170b1fded h1:2J9lQKt+f8NUtyvEmxROEGgz/kTnPoUuVhKexM289J8= +github.com/elastic/apm-data v0.1.1-0.20230727133025-e3e170b1fded/go.mod h1:lMTMoCWNadiDJih/tLechuMTtumEeedtKJlBOYAv030= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230727143857-7fd27bf1f6c4 h1:sICugAEku+oUCgX9p5UkZk0UtzM0yatddqz0ufEObmE= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20230727143857-7fd27bf1f6c4/go.mod h1:4vXAyKpsDLbuLh5WpMEQDi87RNpVFUxNqj4rrY2TaLg= github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+PiWyZ4I4F4LqrCYgYlNc= diff --git a/internal/beater/processors.go b/internal/beater/processors.go index c0c4c4dd850..7bfdac017d4 100644 --- a/internal/beater/processors.go +++ b/internal/beater/processors.go @@ -27,6 +27,7 @@ import ( "go.elastic.co/fastjson" + "github.com/elastic/apm-data/model/modeljson" "github.com/elastic/apm-data/model/modelpb" "github.com/elastic/apm-server/internal/beater/auth" "github.com/elastic/apm-server/internal/beater/ratelimit" @@ -103,7 +104,7 @@ func newDocappenderBatchProcessor(a *docappender.Appender) modelpb.ProcessBatchF return func(ctx context.Context, b *modelpb.Batch) error { for _, event := range *b { r := pool.Get().(*pooledReader) - if err := event.MarshalFastJSON(&r.jsonw); err != nil { + if err := modeljson.MarshalAPMEvent(event, &r.jsonw); err != nil { r.reset() return err } diff --git a/internal/publish/pub.go b/internal/publish/pub.go index 9fc79e8c24c..a975493392d 100644 --- a/internal/publish/pub.go +++ b/internal/publish/pub.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/apm-data/model/modeljson" "github.com/elastic/apm-data/model/modelpb" ) @@ -194,7 +195,7 @@ func (t batchTransformer) Transform(context.Context) []beat.Event { // from the intermediate map representation for events, // and encode directly to JSON, minimising garbage for // the Elasticsearch output. - if err := event.MarshalFastJSON(&w); err != nil { + if err := modeljson.MarshalAPMEvent(event, &w); err != nil { continue } beatEvent := beat.Event{Timestamp: event.Timestamp.AsTime()} diff --git a/systemtest/approvals/TestCompressedSpans.approved.json b/systemtest/approvals/TestCompressedSpans.approved.json index 8cfdd94fc04..79ef04894ca 100644 --- a/systemtest/approvals/TestCompressedSpans.approved.json +++ b/systemtest/approvals/TestCompressedSpans.approved.json @@ -6,9 +6,11 @@ "name": "go", "version": "0.0.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -97,9 +99,11 @@ "name": "go", "version": "0.0.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestDropUnsampled.approved.json b/systemtest/approvals/TestDropUnsampled.approved.json index 32756ebf3ca..b66b9b816a7 100644 --- a/systemtest/approvals/TestDropUnsampled.approved.json +++ b/systemtest/approvals/TestDropUnsampled.approved.json @@ -6,9 +6,11 @@ "name": "go", "version": "0.0.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -52,6 +54,9 @@ "version": "2.0" } }, + "span": { + "id": "dynamic" + }, "timestamp": { "us": "dynamic" }, @@ -82,9 +87,11 @@ "client": { "ip": "127.0.0.1" }, - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -105,6 +112,9 @@ "ip": "127.0.0.1", "port": "dynamic" }, + "span": { + "id": "dynamic" + }, "timestamp": { "us": "dynamic" }, diff --git a/systemtest/approvals/TestErrorIngest.approved.json b/systemtest/approvals/TestErrorIngest.approved.json index 5078ac4763e..9b14c0b3d7a 100644 --- a/systemtest/approvals/TestErrorIngest.approved.json +++ b/systemtest/approvals/TestErrorIngest.approved.json @@ -34,9 +34,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", "grouping_name": "Cannot read property 'baz' of undefined", @@ -111,6 +113,9 @@ }, "version": "5.1.3" }, + "span": { + "id": "1234567890987654" + }, "timestamp": { "us": 1533827045999000 }, diff --git a/systemtest/approvals/TestFleetIntegration.approved.json b/systemtest/approvals/TestFleetIntegration.approved.json deleted file mode 100644 index ec24a4cc0f8..00000000000 --- a/systemtest/approvals/TestFleetIntegration.approved.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "events": [ - { - "@timestamp": "dynamic", - "agent": { - "name": "go", - "version": "0.0.0" - }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "success", - "success_count": 1 - }, - "host": { - "architecture": "i386", - "hostname": "beowulf", - "ip": [ - "10.11.12.13" - ], - "name": "beowulf", - "os": { - "platform": "minix" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "process": { - "pid": 1, - "title": "systemtest.test" - }, - "processor": { - "event": "transaction", - "name": "transaction" - }, - "service": { - "language": { - "name": "go", - "version": "2.0" - }, - "name": "systemtest", - "node": { - "name": "beowulf" - }, - "runtime": { - "name": "gc", - "version": "2.0" - } - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "dynamic" - }, - "transaction": { - "duration": { - "us": 1000000 - }, - "id": "dynamic", - "name": "name", - "representative_count": 1, - "sampled": true, - "span_count": { - "dropped": 0, - "started": 0 - }, - "type": "type" - } - } - ] -} diff --git a/systemtest/approvals/TestFleetIntegrationSourcemap.approved.json b/systemtest/approvals/TestFleetIntegrationSourcemap.approved.json deleted file mode 100644 index 99a821f4b47..00000000000 --- a/systemtest/approvals/TestFleetIntegrationSourcemap.approved.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "events": [ - { - "@timestamp": "dynamic", - "agent": { - "name": "rum-js", - "version": "0.0.0" - }, - "client": { - "ip": "dynamic" - }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", - "error": { - "culprit": "webpack:///webpack/bootstrap 6002740481c9666b0d38 in \u003canonymous\u003e", - "exception": [ - { - "message": "Uncaught Error: timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "__webpack_require__", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/../test/e2e/general-usecase/bundle.js.map", - "colno": 18, - "filename": "test/e2e/general-usecase/bundle.js.map", - "function": "\u003canonymous\u003e", - "library_frame": true, - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t// __webpack_public_path__", - " \t__webpack_require__.p = \"\";", - "", - " \t// Load entry module and return exports" - ], - "pre": [ - "", - " \t// expose the modules object (__webpack_modules__)", - " \t__webpack_require__.m = modules;", - "", - " \t// expose the module cache" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003cunknown\u003e", - "line": { - "column": 0, - "context": " \t__webpack_require__.c = installedModules;", - "number": 33 - }, - "original": { - "abs_path": "http://localhost:8000/test/./e2e/general-usecase/bundle.js.map", - "colno": 181, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invokeTask", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003cunknown\u003e", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 15, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "runTask", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - "", - "", - "/** WEBPACK FOOTER **", - " ** webpack/bootstrap 6002740481c9666b0d38" - ], - "pre": [ - "", - " \t// __webpack_public_path__", - " \t__webpack_require__.p = \"\";", - "", - " \t// Load entry module and return exports" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "moduleId", - "line": { - "column": 0, - "context": " \treturn __webpack_require__(0);", - "number": 39 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 199, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invoke", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - " \t\t\treturn installedModules[moduleId].exports;", - "", - " \t\t// Create a new module (and put it into the cache)", - " \t\tvar module = installedModules[moduleId] = {", - " \t\t\texports: {}," - ], - "pre": [ - "", - " \t// The require function", - " \tfunction __webpack_require__(moduleId) {", - "", - " \t\t// Check if module is in cache" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003canonymous\u003e", - "line": { - "column": 0, - "context": " \t\tif(installedModules[moduleId])", - "number": 8 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 33, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "timer", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - } - ], - "type": "Error" - } - ], - "grouping_key": "89e23da755c2dd759d2d529e37c92b8f", - "grouping_name": "Uncaught Error: log timeout test error", - "id": "aba2688e033848ce9c4e4005f1caa534", - "log": { - "message": "Uncaught Error: log timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003canonymous\u003e", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 18, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "\u003canonymous\u003e", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - } - ] - } - }, - "event": { - "agent_id_status": "missing", - "ingested": "dynamic" - }, - "http": { - "request": { - "referrer": "http://localhost:8000/test/e2e/" - } - }, - "message": "Uncaught Error: log timeout test error", - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "processor": { - "event": "error", - "name": "error" - }, - "service": { - "name": "apm-agent-js", - "version": "1.0.1" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "timestamp": { - "us": "dynamic" - }, - "url": { - "domain": "localhost", - "full": "http://localhost:8000/test/e2e/general-usecase/", - "original": "http://localhost:8000/test/e2e/general-usecase/", - "path": "/test/e2e/general-usecase/", - "port": 8000, - "scheme": "http" - }, - "user_agent": { - "device": { - "name": "Other" - }, - "name": "Go-http-client", - "original": "Go-http-client/1.1", - "version": "1.1" - } - } - ] -} diff --git a/systemtest/approvals/TestIntake/Errors.approved.json b/systemtest/approvals/TestIntake/Errors.approved.json index cafbb6c42e4..105ac8ee7c0 100644 --- a/systemtest/approvals/TestIntake/Errors.approved.json +++ b/systemtest/approvals/TestIntake/Errors.approved.json @@ -46,9 +46,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "culprit": "my.module.function_name", "custom": { @@ -401,9 +403,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -520,9 +524,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "grouping_key": "dc8dd667f7036ec5f0bae87bf2188243", "grouping_name": "no user found", @@ -643,9 +649,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -767,9 +775,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", "grouping_name": "Cannot read property 'baz' of undefined", @@ -844,6 +854,9 @@ }, "version": "5.1.3" }, + "span": { + "id": "1234567890987654" + }, "timestamp": { "us": 1533827045999000 }, diff --git a/systemtest/approvals/TestIntake/ErrorsTxID.approved.json b/systemtest/approvals/TestIntake/ErrorsTxID.approved.json index 317b5a5064f..df69edbb843 100644 --- a/systemtest/approvals/TestIntake/ErrorsTxID.approved.json +++ b/systemtest/approvals/TestIntake/ErrorsTxID.approved.json @@ -13,9 +13,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "culprit": "opbeans.controllers.DTInterceptor.preHandle(DTInterceptor.java:73)", "custom": { @@ -265,6 +267,9 @@ "ip": "12.53.12.1" } }, + "span": { + "id": "1234567890987654" + }, "timestamp": { "us": 1571657444929001 }, diff --git a/systemtest/approvals/TestIntake/Events.approved.json b/systemtest/approvals/TestIntake/Events.approved.json index 177dec39cc8..6d18f573d63 100644 --- a/systemtest/approvals/TestIntake/Events.approved.json +++ b/systemtest/approvals/TestIntake/Events.approved.json @@ -14,9 +14,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "culprit": "opbeans.controllers.DTInterceptor.preHandle(DTInterceptor.java:73)", "custom": { @@ -266,6 +268,9 @@ "ip": "12.53.12.1" } }, + "span": { + "id": "1234567890987654" + }, "timestamp": { "us": 1571657444929001 }, @@ -410,9 +415,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -585,9 +592,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -718,6 +727,9 @@ "ip": "12.53.12.1", "port": 8080 }, + "span": { + "id": "4340a8e0df1906ecbfa9" + }, "timestamp": { "us": 1571657444929001 }, diff --git a/systemtest/approvals/TestIntake/MinimalEvents.approved.json b/systemtest/approvals/TestIntake/MinimalEvents.approved.json index 90de415fdda..fb27a47701d 100644 --- a/systemtest/approvals/TestIntake/MinimalEvents.approved.json +++ b/systemtest/approvals/TestIntake/MinimalEvents.approved.json @@ -6,9 +6,11 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "grouping_key": "0b9cba09845a097a271c6beb4c6207f3", "grouping_name": "error log message", @@ -45,9 +47,11 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -86,9 +90,11 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -164,9 +170,11 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -213,9 +221,11 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -262,9 +272,11 @@ "name": "elastic-node", "version": "3.14.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -286,6 +298,9 @@ "service": { "name": "1234_service-12a3" }, + "span": { + "id": "abcdef1478523690" + }, "timestamp": { "us": "dynamic" }, diff --git a/systemtest/approvals/TestIntake/Spans.approved.json b/systemtest/approvals/TestIntake/Spans.approved.json index dfc6104874e..d5b4e708fa6 100644 --- a/systemtest/approvals/TestIntake/Spans.approved.json +++ b/systemtest/approvals/TestIntake/Spans.approved.json @@ -34,9 +34,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "0:0::0:1", "ip": "0:0::0:1", @@ -255,9 +257,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -387,9 +391,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -525,9 +531,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -657,9 +665,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -800,9 +810,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "0:0::0:1", "ip": "0:0::0:1", @@ -1021,9 +1033,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "0:0::0:1", "ip": "0:0::0:1" @@ -1173,9 +1187,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -1308,9 +1324,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "0:0::0:1", "ip": "0:0::0:1", @@ -1460,9 +1478,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "0:0::0:1", "ip": "0:0::0:1", diff --git a/systemtest/approvals/TestIntake/Transactions.approved.json b/systemtest/approvals/TestIntake/Transactions.approved.json index a2768432ab5..94803e73240 100644 --- a/systemtest/approvals/TestIntake/Transactions.approved.json +++ b/systemtest/approvals/TestIntake/Transactions.approved.json @@ -33,9 +33,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -114,6 +116,9 @@ "id": "sunday", "sequence": 123 }, + "span": { + "id": "00xxxxFFaaaa1234" + }, "timestamp": { "us": 1496170407154000 }, @@ -191,9 +196,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -268,6 +275,9 @@ }, "version": "5.1.3" }, + "span": { + "id": "945254c567a5417e" + }, "timestamp": { "us": 1496170407154000 }, @@ -337,9 +347,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -475,6 +487,9 @@ "source": { "ip": "12.53.12.1" }, + "span": { + "id": "4340a8e0df1906ecbfa9" + }, "timestamp": { "us": 1496170407154000 }, @@ -569,9 +584,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -649,6 +666,9 @@ }, "version": "2" }, + "span": { + "id": "cdef4340a8e0df19" + }, "timestamp": { "us": 1532976822281000 }, @@ -737,9 +757,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -826,6 +848,9 @@ }, "version": "5.1.3" }, + "span": { + "id": "142e61450efb8574" + }, "timestamp": { "us": 1631736666365048 }, diff --git a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json index 7b9c5ae8783..2744ecfa04f 100644 --- a/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json +++ b/systemtest/approvals/TestIntake/TransactionsHugeTraces.approved.json @@ -45,9 +45,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -183,6 +185,9 @@ "source": { "ip": "12.53.12.1" }, + "span": { + "id": "ddf109a4c4aa5f2b6e984548ca57774c" + }, "timestamp": { "us": 1496170407154000 }, diff --git a/systemtest/approvals/TestIntake/UnknownSpanType.approved.json b/systemtest/approvals/TestIntake/UnknownSpanType.approved.json index 77e1c7174c9..15e8afae371 100644 --- a/systemtest/approvals/TestIntake/UnknownSpanType.approved.json +++ b/systemtest/approvals/TestIntake/UnknownSpanType.approved.json @@ -9,9 +9,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -168,9 +170,11 @@ "container": { "id": "container-id" }, - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -290,6 +294,9 @@ "version": "8.0.0" } }, + "span": { + "id": "945254c567a5417e" + }, "timestamp": { "us": 1496170407154000 }, diff --git a/systemtest/approvals/TestIntakeLog/with_faas.approved.json b/systemtest/approvals/TestIntakeLog/with_faas.approved.json index 9cf823957bd..ed47f3d7abe 100644 --- a/systemtest/approvals/TestIntakeLog/with_faas.approved.json +++ b/systemtest/approvals/TestIntakeLog/with_faas.approved.json @@ -11,9 +11,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.app.1234_service_12a3", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.1234_service_12a3", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestIntakeLog/with_flat_ecs_fields.approved.json b/systemtest/approvals/TestIntakeLog/with_flat_ecs_fields.approved.json index e9048ca81bd..27c8b5e47ae 100644 --- a/systemtest/approvals/TestIntakeLog/with_flat_ecs_fields.approved.json +++ b/systemtest/approvals/TestIntakeLog/with_flat_ecs_fields.approved.json @@ -11,9 +11,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.app.testsvc", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.testsvc", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "dataset": "accesslog", @@ -107,6 +109,9 @@ }, "version": "v1.0.0" }, + "span": { + "id": "txn-id" + }, "trace": { "id": "trace-id" }, diff --git a/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields.approved.json b/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields.approved.json index 7eb2973fe91..99841d0e0be 100644 --- a/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields.approved.json +++ b/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields.approved.json @@ -11,9 +11,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.app.testsvc", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.testsvc", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "dataset": "accesslog", @@ -107,6 +109,9 @@ }, "version": "v1.0.0" }, + "span": { + "id": "txn-id" + }, "trace": { "id": "trace-id" }, diff --git a/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields_overrides_flat_fields.approved.json b/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields_overrides_flat_fields.approved.json index 2c9bb947c04..c8fec2ec92e 100644 --- a/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields_overrides_flat_fields.approved.json +++ b/systemtest/approvals/TestIntakeLog/with_nested_ecs_fields_overrides_flat_fields.approved.json @@ -11,9 +11,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.app.testsvc", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.testsvc", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "dataset": "accesslog", @@ -107,6 +109,9 @@ }, "version": "v1.0.0" }, + "span": { + "id": "txn-id" + }, "trace": { "id": "trace-id" }, diff --git a/systemtest/approvals/TestIntakeLog/with_timestamp.approved.json b/systemtest/approvals/TestIntakeLog/with_timestamp.approved.json index 65715103bbe..4cd42b513cd 100644 --- a/systemtest/approvals/TestIntakeLog/with_timestamp.approved.json +++ b/systemtest/approvals/TestIntakeLog/with_timestamp.approved.json @@ -11,9 +11,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.app.1234_service_12a3", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.1234_service_12a3", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestIntakeLog/with_timestamp_as_str.approved.json b/systemtest/approvals/TestIntakeLog/with_timestamp_as_str.approved.json index f831d915757..15ef9019605 100644 --- a/systemtest/approvals/TestIntakeLog/with_timestamp_as_str.approved.json +++ b/systemtest/approvals/TestIntakeLog/with_timestamp_as_str.approved.json @@ -11,9 +11,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.app.1234_service_12a3", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.1234_service_12a3", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestIntakeLog/without_timestamp.approved.json b/systemtest/approvals/TestIntakeLog/without_timestamp.approved.json index 81686dc4085..bc9772d99e4 100644 --- a/systemtest/approvals/TestIntakeLog/without_timestamp.approved.json +++ b/systemtest/approvals/TestIntakeLog/without_timestamp.approved.json @@ -11,9 +11,11 @@ "container": { "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, - "data_stream.dataset": "apm.app.1234_service_12a3", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.1234_service_12a3", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestJaeger/batch_0.approved.json b/systemtest/approvals/TestJaeger/batch_0.approved.json index d28693b89dd..289d434d87c 100644 --- a/systemtest/approvals/TestJaeger/batch_0.approved.json +++ b/systemtest/approvals/TestJaeger/batch_0.approved.json @@ -7,9 +7,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -51,6 +53,9 @@ "name": "host01" } }, + "span": { + "id": "7be2fd98d0973be3" + }, "timestamp": { "us": 1576827705007552 }, @@ -70,9 +75,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -114,6 +121,9 @@ "name": "host01" } }, + "span": { + "id": "7be2fd98d0973be3" + }, "timestamp": { "us": 1576827705089431 }, @@ -133,9 +143,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -177,6 +189,9 @@ "name": "host01" } }, + "span": { + "id": "7be2fd98d0973be3" + }, "timestamp": { "us": 1576827705172530 }, @@ -196,9 +211,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.app.driver", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.driver", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -232,6 +249,9 @@ "name": "host01" } }, + "span": { + "id": "7be2fd98d0973be3" + }, "trace": { "id": "00000000000000007be2fd98d0973be3" }, @@ -246,9 +266,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.app.driver", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.driver", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -280,6 +302,9 @@ "name": "host01" } }, + "span": { + "id": "7be2fd98d0973be3" + }, "trace": { "id": "00000000000000007be2fd98d0973be3" }, @@ -294,9 +319,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.app.driver", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.driver", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -328,6 +355,9 @@ "name": "host01" } }, + "span": { + "id": "7be2fd98d0973be3" + }, "trace": { "id": "00000000000000007be2fd98d0973be3" }, @@ -342,9 +372,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -384,6 +416,9 @@ "name": "host01" } }, + "span": { + "id": "7be2fd98d0973be3" + }, "timestamp": { "us": 1576827704953864 }, diff --git a/systemtest/approvals/TestJaeger/batch_1.approved.json b/systemtest/approvals/TestJaeger/batch_1.approved.json index e591f44d64b..f25f9842f8a 100644 --- a/systemtest/approvals/TestJaeger/batch_1.approved.json +++ b/systemtest/approvals/TestJaeger/batch_1.approved.json @@ -7,9 +7,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -68,9 +70,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -129,9 +133,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -190,9 +196,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm.app.redis", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.redis", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -240,9 +248,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -302,9 +312,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -363,9 +375,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -424,9 +438,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -485,9 +501,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -547,9 +565,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -608,9 +628,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -670,9 +692,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -731,9 +755,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -792,9 +818,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -853,9 +881,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -914,9 +944,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -975,9 +1007,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -1036,9 +1070,11 @@ "name": "Jaeger/Go", "version": "2.20.1" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestNoMatchingSourcemap.approved.json b/systemtest/approvals/TestNoMatchingSourcemap.approved.json index 0749bc0f369..41a9c946f57 100644 --- a/systemtest/approvals/TestNoMatchingSourcemap.approved.json +++ b/systemtest/approvals/TestNoMatchingSourcemap.approved.json @@ -9,9 +9,11 @@ "client": { "ip": "127.0.0.1" }, - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestOTLPGRPCLogs.approved.json b/systemtest/approvals/TestOTLPGRPCLogs.approved.json index 7469014acbc..3c8e7de4497 100644 --- a/systemtest/approvals/TestOTLPGRPCLogs.approved.json +++ b/systemtest/approvals/TestOTLPGRPCLogs.approved.json @@ -6,9 +6,11 @@ "name": "otlp/go", "version": "unknown" }, - "data_stream.dataset": "apm.app.unknown", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.unknown", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestOTLPGRPCTraces.approved.json b/systemtest/approvals/TestOTLPGRPCTraces.approved.json index 1be600ec06a..9f74d39ca58 100644 --- a/systemtest/approvals/TestOTLPGRPCTraces.approved.json +++ b/systemtest/approvals/TestOTLPGRPCTraces.approved.json @@ -6,9 +6,11 @@ "name": "opentelemetry/go", "version": "1.16.0" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "exception": [ { @@ -63,6 +65,9 @@ }, "name": "unknown_service_systemtest_test" }, + "span": { + "id": "b3ee9be3b687a611" + }, "timestamp": { "us": 123002000 }, @@ -81,9 +86,11 @@ "name": "opentelemetry/go", "version": "1.16.0" }, - "data_stream.dataset": "apm.app.unknown_service_systemtest_test", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.app.unknown_service_systemtest_test", + "namespace": "default", + "type": "logs" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -123,6 +130,9 @@ }, "name": "unknown_service_systemtest_test" }, + "span": { + "id": "b3ee9be3b687a611" + }, "trace": { "id": "d2acbef8b37655e48548fd9d61ad6114" }, @@ -136,9 +146,11 @@ "name": "opentelemetry/go", "version": "1.16.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -185,6 +197,9 @@ }, "name": "unknown_service_systemtest_test" }, + "span": { + "id": "b3ee9be3b687a611" + }, "timestamp": { "us": 123000000 }, diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json index fa2222f91a1..a677990e718 100644 --- a/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json +++ b/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json @@ -9,9 +9,11 @@ "client": { "ip": "dynamic" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "culprit": "webpack:///webpack/bootstrap 6002740481c9666b0d38 in \u003canonymous\u003e", "exception": [ diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json index fa2222f91a1..a677990e718 100644 --- a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json +++ b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json @@ -9,9 +9,11 @@ "client": { "ip": "dynamic" }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", + "data_stream": { + "dataset": "apm.error", + "namespace": "default", + "type": "logs" + }, "error": { "culprit": "webpack:///webpack/bootstrap 6002740481c9666b0d38 in \u003canonymous\u003e", "exception": [ diff --git a/systemtest/approvals/TestRUMRoutingIntegration.approved.json b/systemtest/approvals/TestRUMRoutingIntegration.approved.json index 79e8c5fd50c..7a84bbcfcc9 100644 --- a/systemtest/approvals/TestRUMRoutingIntegration.approved.json +++ b/systemtest/approvals/TestRUMRoutingIntegration.approved.json @@ -7,9 +7,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "localhost", "port": 8003 @@ -91,9 +93,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "localhost", "port": 8000 @@ -175,9 +179,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -235,9 +241,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "localhost", "port": 8003 @@ -320,9 +328,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -381,9 +391,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -470,9 +482,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "destination": { "address": "localhost", "port": 8000 @@ -552,9 +566,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -613,9 +629,11 @@ "version": "4.8.1" }, "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestRUMSpanSourcemapping.approved.json b/systemtest/approvals/TestRUMSpanSourcemapping.approved.json index 40cb3dc9fc4..fb48b344364 100644 --- a/systemtest/approvals/TestRUMSpanSourcemapping.approved.json +++ b/systemtest/approvals/TestRUMSpanSourcemapping.approved.json @@ -9,9 +9,11 @@ "client": { "ip": "127.0.0.1" }, - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", diff --git a/systemtest/approvals/TestRUMXForwardedFor.approved.json b/systemtest/approvals/TestRUMXForwardedFor.approved.json index e71198ad00e..be7db099118 100644 --- a/systemtest/approvals/TestRUMXForwardedFor.approved.json +++ b/systemtest/approvals/TestRUMXForwardedFor.approved.json @@ -92,9 +92,11 @@ }, "ip": "220.244.41.16" }, - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm.rum", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -117,6 +119,9 @@ "ip": "127.0.0.1" } }, + "span": { + "id": "611f4fa950f04631" + }, "timestamp": { "us": "dynamic" }, diff --git a/systemtest/approvals/TestTransactionDroppedSpansStatsTransaction.approved.json b/systemtest/approvals/TestTransactionDroppedSpansStatsTransaction.approved.json index 30fa07e2d3a..d05aa17e1ad 100644 --- a/systemtest/approvals/TestTransactionDroppedSpansStatsTransaction.approved.json +++ b/systemtest/approvals/TestTransactionDroppedSpansStatsTransaction.approved.json @@ -6,9 +6,11 @@ "name": "go", "version": "0.0.0" }, - "data_stream.dataset": "apm", - "data_stream.namespace": "default", - "data_stream.type": "traces", + "data_stream": { + "dataset": "apm", + "namespace": "default", + "type": "traces" + }, "event": { "agent_id_status": "missing", "ingested": "dynamic", @@ -52,6 +54,9 @@ "version": "2.0" } }, + "span": { + "id": "dynamic" + }, "timestamp": "dynamic", "trace": { "id": "dynamic" diff --git a/systemtest/huge_traces_test.go b/systemtest/huge_traces_test.go index beee9a42c1d..771784b208e 100644 --- a/systemtest/huge_traces_test.go +++ b/systemtest/huge_traces_test.go @@ -82,7 +82,7 @@ func TestTransactionDroppedSpansStats(t *testing.T) { espoll.TermQuery{Field: "transaction.id", Value: tx.TraceContext().Span.String()}, ) approvaltest.ApproveEvents(t, t.Name()+"Transaction", txResult.Hits.Hits, - "@timestamp", "timestamp", "trace.id", "transaction.id", + "@timestamp", "timestamp", "trace.id", "transaction.id", "span.id", ) } diff --git a/systemtest/sampling_test.go b/systemtest/sampling_test.go index 00208d9f5d5..6ac2521d8a3 100644 --- a/systemtest/sampling_test.go +++ b/systemtest/sampling_test.go @@ -80,7 +80,7 @@ func TestDropUnsampled(t *testing.T) { // RUM events have the source port recorded, and in the tests it will be dynamic "source.port", // Ignore dynamically generated trace/transaction ID - "trace.id", "transaction.id", + "trace.id", "transaction.id", "span.id", ) doc := getBeatsMonitoringStats(t, srv, nil) diff --git a/x-pack/apm-server/sampling/pubsub/pubsub.go b/x-pack/apm-server/sampling/pubsub/pubsub.go index e4eda7a61ce..ed6886ab113 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/pkg/errors" + "go.elastic.co/fastjson" "go.uber.org/zap" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/types/known/timestamppb" @@ -25,6 +26,7 @@ import ( "github.com/elastic/go-elasticsearch/v8/esapi" "github.com/elastic/go-elasticsearch/v8/esutil" + "github.com/elastic/apm-data/model/modeljson" "github.com/elastic/apm-data/model/modelpb" "github.com/elastic/apm-server/internal/logs" ) @@ -97,6 +99,7 @@ func (p *Pubsub) indexSampledTraceIDs(ctx context.Context, traceIDs <-chan strin if !ok { return nil } + var w fastjson.Writer doc := modelpb.APMEvent{ Timestamp: timestamppb.New(time.Now()), DataStream: &modelpb.DataStream{ @@ -107,7 +110,7 @@ func (p *Pubsub) indexSampledTraceIDs(ctx context.Context, traceIDs <-chan strin Agent: &modelpb.Agent{EphemeralId: p.config.ServerID}, Trace: &modelpb.Trace{Id: id}, } - data, err := doc.MarshalJSON() + err := modeljson.MarshalAPMEvent(&doc, &w) if err != nil { p.config.Logger.With( logp.Error(err), @@ -115,6 +118,7 @@ func (p *Pubsub) indexSampledTraceIDs(ctx context.Context, traceIDs <-chan strin ).Debug("failed to encode sampled trace document") return err } + data := w.Bytes() if err := appender.Add(ctx, index, bytes.NewReader(data)); err != nil { p.config.Logger.With( logp.Error(err), diff --git a/x-pack/apm-server/sampling/pubsub/pubsub_test.go b/x-pack/apm-server/sampling/pubsub/pubsub_test.go index 03f5b509f39..5bb73f6b538 100644 --- a/x-pack/apm-server/sampling/pubsub/pubsub_test.go +++ b/x-pack/apm-server/sampling/pubsub/pubsub_test.go @@ -99,9 +99,11 @@ func TestPublishSampledTraceIDs(t *testing.T) { assert.NoError(t, d.Decode(&doc)) assert.Contains(t, doc, "@timestamp") assert.Equal(t, map[string]interface{}{"ephemeral_id": serverID}, doc["agent"]) - assert.Equal(t, dataStream.Type, doc["data_stream.type"]) - assert.Equal(t, dataStream.Dataset, doc["data_stream.dataset"]) - assert.Equal(t, dataStream.Namespace, doc["data_stream.namespace"]) + ds, ok := doc["data_stream"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, dataStream.Type, ds["type"]) + assert.Equal(t, dataStream.Dataset, ds["dataset"]) + assert.Equal(t, dataStream.Namespace, ds["namespace"]) trace := doc["trace"].(map[string]interface{}) traceID := trace["id"].(string) @@ -110,9 +112,7 @@ func TestPublishSampledTraceIDs(t *testing.T) { assert.Empty(t, trace) // no other fields in "trace" delete(doc, "@timestamp") - delete(doc, "data_stream.type") - delete(doc, "data_stream.dataset") - delete(doc, "data_stream.namespace") + delete(doc, "data_stream") delete(doc, "agent") delete(doc, "trace")