Fix ndjson parser to store JSON fields correctly under target (#36089)

* Fix ndjson parser to store JSON fields correctly under target * Update CHANGELOG.next.asciidoc * Move the change log entry to the right section --------- Co-authored-by: Shaunak Kashyap <[email protected]>
elastic · Jul 20, 2023 · 2aba00a · 2aba00a
1 parent 8a9d6f6
commit 2aba00a
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -74,6 +74,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix recovering from invalid output configuration when running under Elastic-Agent {pull}36016[36016]
 - Improve StreamBuf append to improve performance when reading long lines from files. {pull}35928[35928]
 - Eliminate cloning of event in deepUpdate {pull}35945[35945]
+- Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395]
 - Support build of projects outside of beats directory {pull}36126[36126]
 
 *Auditbeat*

diff --git a/libbeat/reader/parser/parser_test.go b/libbeat/reader/parser/parser_test.go
@@ -367,6 +367,54 @@ func TestJSONParsersWithFields(t *testing.T) {
 				},
 			},
 		},
+		"JSON post processor with dotted target key": {
+			message: reader.Message{
+				Content: []byte("{\"key\":\"value\"}"),
+				Fields:  mapstr.M{},
+			},
+			config: map[string]interface{}{
+				"parsers": []map[string]interface{}{
+					map[string]interface{}{
+						"ndjson": map[string]interface{}{
+							"target": "kubernetes.audit",
+						},
+					},
+				},
+			},
+			expectedMessage: reader.Message{
+				Content: []byte(""),
+				Fields: mapstr.M{
+					"kubernetes": mapstr.M{
+						"audit": mapstr.M{
+							"key": "value",
+						},
+					},
+				},
+			},
+		},
+		"JSON post processor with non-dotted target key": {
+			message: reader.Message{
+				Content: []byte("{\"key\":\"value\"}"),
+				Fields:  mapstr.M{},
+			},
+			config: map[string]interface{}{
+				"parsers": []map[string]interface{}{
+					map[string]interface{}{
+						"ndjson": map[string]interface{}{
+							"target": "kubernetes",
+						},
+					},
+				},
+			},
+			expectedMessage: reader.Message{
+				Content: []byte(""),
+				Fields: mapstr.M{
+					"kubernetes": mapstr.M{
+						"key": "value",
+					},
+				},
+			},
+		},
 		"JSON post processor with document ID": {
 			message: reader.Message{
 				Content: []byte("{\"key\":\"value\", \"my-id-field\":\"my-id\"}"),

diff --git a/libbeat/reader/readjson/json.go b/libbeat/reader/readjson/json.go
@@ -195,7 +195,9 @@ func (p *JSONParser) Next() (reader.Message, error) {
 		message.Fields = event.Fields
 		message.Meta = event.Meta
 	} else {
-		message.AddFields(mapstr.M{p.target: jsonFields})
+		fields := mapstr.M{}
+		fields.Put(p.target, jsonFields)
+		message.AddFields(fields)
 	}
 
 	return message, err