diff --git a/core/models/PipelineEventGroup.cpp b/core/models/PipelineEventGroup.cpp index fdf2c206ae..c5482e5fda 100644 --- a/core/models/PipelineEventGroup.cpp +++ b/core/models/PipelineEventGroup.cpp @@ -16,6 +16,8 @@ #include "models/PipelineEventGroup.h" +#include + #include #include "logger/Logger.h" @@ -170,6 +172,9 @@ const std::string EVENT_GROUP_META_CONTAINER_IP = "container.ip"; const std::string EVENT_GROUP_META_CONTAINER_IMAGE_NAME = "container.image.name"; const std::string EVENT_GROUP_META_CONTAINER_IMAGE_ID = "container.image.id"; +const std::string EVENT_GROUP_META_CONTAINERD_TEXT = "containerd_text"; +const std::string EVENT_GROUP_META_DOCKER_JSON_FILE = "docker_json-file"; + const std::string& EventGroupMetaKeyToString(EventGroupMetaKey key) { switch (key) { case EventGroupMetaKey::AGENT_TAG: @@ -198,6 +203,15 @@ const std::string& EventGroupMetaKeyToString(EventGroupMetaKey key) { } } +const std::string EventGroupMetaValueToString(std::string value) { + if (value == ProcessorParseContainerLogNative::CONTAINERD_TEXT) { + return EVENT_GROUP_META_CONTAINERD_TEXT; + } else if (value == ProcessorParseContainerLogNative::DOCKER_JSON_FILE) { + return EVENT_GROUP_META_DOCKER_JSON_FILE; + } + return value; +} + EventGroupMetaKey StringToEventGroupMetaKey(const std::string& key) { static std::unordered_map sStringToEnum = {{EVENT_GROUP_META_AGENT_TAG, EventGroupMetaKey::AGENT_TAG}, @@ -221,7 +235,7 @@ Json::Value PipelineEventGroup::ToJson() const { if (!mMetadata.empty()) { Json::Value metadata; for (const auto& meta : mMetadata) { - metadata[EventGroupMetaKeyToString(meta.first)] = meta.second.to_string(); + metadata[EventGroupMetaKeyToString(meta.first)] = EventGroupMetaValueToString(meta.second.to_string()); } root["metadata"] = metadata; } diff --git a/core/processor/ProcessorParseContainerLogNative.cpp b/core/processor/ProcessorParseContainerLogNative.cpp index a2d8ba6ba5..f5c12fb786 100644 --- a/core/processor/ProcessorParseContainerLogNative.cpp +++ b/core/processor/ProcessorParseContainerLogNative.cpp @@ -27,6 +27,9 @@ namespace logtail { +const std::string ProcessorParseContainerLogNative::CONTAINERD_TEXT = "1"; +const std::string ProcessorParseContainerLogNative::DOCKER_JSON_FILE = "2"; + const std::string ProcessorParseContainerLogNative::sName = "processor_parse_container_log_native"; const char ProcessorParseContainerLogNative::CONTIANERD_DELIMITER = ' '; // 分隔符 const char ProcessorParseContainerLogNative::CONTIANERD_FULL_TAG = 'F'; // 容器全标签 @@ -133,9 +136,9 @@ bool ProcessorParseContainerLogNative::ProcessEvent(StringView containerType, Pi std::string errorMsg; bool shouldKeepEvent = true; - if (containerType == "containerd_text") { + if (containerType == CONTAINERD_TEXT) { shouldKeepEvent = ParseContainerdTextLogLine(sourceEvent, errorMsg); - } else if (containerType == "docker_json-file") { + } else if (containerType == DOCKER_JSON_FILE) { shouldKeepEvent = ParseDockerJsonLogLine(sourceEvent, errorMsg); } if (!errorMsg.empty()) { @@ -250,22 +253,20 @@ bool ProcessorParseContainerLogNative::ParseDockerJsonLogLine(LogEvent& sourceEv return true; } - auto findMemberAndGetString = [&](const std::string& memberName) { - auto it = doc.FindMember(memberName.c_str()); - return it != doc.MemberEnd() ? StringView(it->value.GetString()) : StringView(); - }; - - StringView timeValue = findMemberAndGetString(DOCKER_JSON_TIME); - if (timeValue.empty()) { + // time + auto it = doc.FindMember(DOCKER_JSON_TIME.c_str()); + if (it == doc.MemberEnd() || !it->value.IsString()) { std::ostringstream errorMsgStream; errorMsgStream << "time field cannot be found in log line." << "\tfirst 1KB log:" << buffer.substr(0, 1024).to_string(); errorMsg = errorMsgStream.str(); return true; } + StringView timeValue = StringView(it->value.GetString()); - auto it = doc.FindMember(DOCKER_JSON_LOG.c_str()); - if (it == doc.MemberEnd()) { + // content + it = doc.FindMember(DOCKER_JSON_LOG.c_str()); + if (it == doc.MemberEnd() || !it->value.IsString()) { std::ostringstream errorMsgStream; errorMsgStream << "content field cannot be found in log line." << "\tfirst 1KB log:" << buffer.substr(0, 1024).to_string(); @@ -274,7 +275,14 @@ bool ProcessorParseContainerLogNative::ParseDockerJsonLogLine(LogEvent& sourceEv } StringView content = StringView(it->value.GetString()); - StringView sourceValue = findMemberAndGetString(DOCKER_JSON_STREAM_TYPE); + // source + it = doc.FindMember(DOCKER_JSON_STREAM_TYPE.c_str()); + StringView sourceValue; + if (it == doc.MemberEnd() || !it->value.IsString()) { + sourceValue = StringView(); + } else { + sourceValue = StringView(it->value.GetString()); + } if (sourceValue.empty() || (sourceValue != "stdout" && sourceValue != "stderr")) { std::ostringstream errorMsgStream; errorMsgStream << "source field cannot be found in log line." diff --git a/core/processor/ProcessorParseContainerLogNative.h b/core/processor/ProcessorParseContainerLogNative.h index 81f2a63d82..1038787acc 100644 --- a/core/processor/ProcessorParseContainerLogNative.h +++ b/core/processor/ProcessorParseContainerLogNative.h @@ -25,6 +25,9 @@ class ProcessorParseContainerLogNative : public Processor { public: static const std::string sName; + static const std::string CONTAINERD_TEXT; + static const std::string DOCKER_JSON_FILE; + // needed by LastMatchedLine static const char CONTIANERD_DELIMITER; // 分隔符 static const char CONTIANERD_FULL_TAG; // 容器全标签 @@ -60,7 +63,7 @@ class ProcessorParseContainerLogNative : public Processor { bool ParseDockerJsonLogLine(LogEvent& sourceEvent, std::string& errorMsg); CounterPtr mProcParseInSizeBytes; // 成功且保留的日志中，解析字段的INBYTES - CounterPtr mProcParseOutSizeBytes; // 成功且保留的日志中，解析出来字段的OUTBYTES和 + CounterPtr mProcParseOutSizeBytes; // 成功且保留的日志中，解析出来字段的OUTBYTES CounterPtr mProcParseErrorTotal; // 解析失败条数 CounterPtr mProcParseSuccessTotal; // 成功解析条数 // CounterPtr mProcParseSuccessSizeBytes; // 成功bytes diff --git a/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp b/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp index b5a796b2e6..c704e86f23 100644 --- a/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseContainerLogNativeUnittest.cpp @@ -154,8 +154,7 @@ void ProcessorParseContainerLogNativeUnittest::TestIgnoringStdoutStderr() { auto sourceBuffer = std::make_shared(); { PipelineEventGroup eventGroup(sourceBuffer); - std::string containerType = "containerd_text"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::CONTAINERD_TEXT); std::string inJson = R"({ "events" : [ @@ -261,8 +260,7 @@ void ProcessorParseContainerLogNativeUnittest::TestIgnoringStdoutStderr() { auto sourceBuffer = std::make_shared(); { PipelineEventGroup eventGroup(sourceBuffer); - std::string containerType = "containerd_text"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::CONTAINERD_TEXT); std::string inJson = R"({ "events" : [ @@ -400,8 +398,7 @@ void ProcessorParseContainerLogNativeUnittest::TestIgnoringStdoutStderr() { auto sourceBuffer = std::make_shared(); { PipelineEventGroup eventGroup(sourceBuffer); - std::string containerType = "containerd_text"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::CONTAINERD_TEXT); std::string inJson = R"({ "events" : [ @@ -562,8 +559,7 @@ void ProcessorParseContainerLogNativeUnittest::TestIgnoringStdoutStderr() { auto sourceBuffer = std::make_shared(); { PipelineEventGroup eventGroup(sourceBuffer); - std::string containerType = "containerd_text"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::CONTAINERD_TEXT); std::string inJson = R"({ "events" : [ @@ -758,8 +754,7 @@ void ProcessorParseContainerLogNativeUnittest::TestContainerdLog() { // case4: 第二个空格不存在 // case5: 第一个空格不存在 PipelineEventGroup eventGroup(sourceBuffer); - std::string containerType = "containerd_text"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::CONTAINERD_TEXT); std::string inJson = R"({ "events" : [ @@ -880,8 +875,7 @@ void ProcessorParseContainerLogNativeUnittest::TestContainerdLog() { { // case：正常情况下的日志 PipelineEventGroup eventGroup(sourceBuffer); - std::string containerType = "containerd_text"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::CONTAINERD_TEXT); std::string inJson = R"({ "events" : [ @@ -992,8 +986,7 @@ void ProcessorParseContainerLogNativeUnittest::TestContainerdLogWithSplit() { auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "containerd_text"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::CONTAINERD_TEXT); inJson << R"({ "events" : [ @@ -1107,8 +1100,7 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParserWithSp auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "docker_json-file"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); inJson << R"({ "events": [ { @@ -1231,8 +1223,7 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "docker_json-file"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); inJson << R"({ "events": [ { @@ -1275,8 +1266,7 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "docker_json-file"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); inJson << R"({ "events": [ { @@ -1316,6 +1306,49 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); } + // log为不是string + { + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ + "events": [ + { + "contents": { + "content": "{\"log\":1,\"stream\":\"stdout\",\"time\":\"2024-02-19T03:49:37.793559367Z\"}" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + eventGroup.FromJsonString(inJson.str()); + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ + "events" : + [ + { + "contents" : + { + "content": "{\"log\":1,\"stream\":\"stdout\",\"time\":\"2024-02-19T03:49:37.793559367Z\"}" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + } + ], + "metadata":{ + "container.type":"docker_json-file" + } + })"; + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } } // time { @@ -1325,8 +1358,7 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "docker_json-file"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); inJson << R"({ "events": [ { @@ -1371,8 +1403,7 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "docker_json-file"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); inJson << R"({ "events": [ { @@ -1411,6 +1442,51 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); } + // time 不是string + { + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ + "events": [ + { + "contents": { + "content": "{\"log\":\"Exception in thread \\\"main\\\" java.lang.NullPoinntterException\\n\",\"stream\":\"stdout\",\"time\":\"2024-02-19T03:49:37.793533014Z\"}\n{\"log\":\" at com.example.myproject.Book.getTitle\\n\",\"stream\":\"stdout\",\"time\":1}" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + eventGroup.FromJsonString(inJson.str()); + + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ + "events" : + [ + { + "contents" : + { + "content": "{\"log\":\"Exception in thread \\\"main\\\" java.lang.NullPoinntterException\\n\",\"stream\":\"stdout\",\"time\":\"2024-02-19T03:49:37.793533014Z\"}\n{\"log\":\" at com.example.myproject.Book.getTitle\\n\",\"stream\":\"stdout\",\"time\":1}" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + } + ], + "metadata":{ + "container.type":"docker_json-file" + } + })"; + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } } // stream { @@ -1420,8 +1496,7 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "docker_json-file"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); inJson << R"({ "events": [ { @@ -1466,8 +1541,7 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { auto sourceBuffer = std::make_shared(); PipelineEventGroup eventGroup(sourceBuffer); std::stringstream inJson; - std::string containerType = "docker_json-file"; - eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, containerType); + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); inJson << R"({ "events": [ { @@ -1506,6 +1580,51 @@ void ProcessorParseContainerLogNativeUnittest::TestDockerJsonLogLineParser() { std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); } + // stream不是string + { + // make eventGroup + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::stringstream inJson; + eventGroup.SetMetadata(EventGroupMetaKey::LOG_FORMAT, ProcessorParseContainerLogNative::DOCKER_JSON_FILE); + inJson << R"({ + "events": [ + { + "contents": { + "content": "{\"log\":\"Exception in thread \\\"main\\\" java.lang.NullPoinntterException\\n\",\"stream\":1,\"time\":\"2024-02-19T03:49:37.793533014Z\"}\n{\"log\":\" at com.example.myproject.Book.getTitle\\n\",\"stream\":\"std\",\"time\":\"2024-02-19T03:49:37.793559367Z\"}" + }, + "timestamp": 12345678901, + "timestampNanosecond": 0, + "type": 1 + } + ] + })"; + eventGroup.FromJsonString(inJson.str()); + + // run test function + processor.Process(eventGroup); + // judge result + std::stringstream expectJson; + expectJson << R"({ + "events" : + [ + { + "contents" : + { + "content": "{\"log\":\"Exception in thread \\\"main\\\" java.lang.NullPoinntterException\\n\",\"stream\":1,\"time\":\"2024-02-19T03:49:37.793533014Z\"}\n{\"log\":\" at com.example.myproject.Book.getTitle\\n\",\"stream\":\"std\",\"time\":\"2024-02-19T03:49:37.793559367Z\"}" + }, + "timestamp" : 12345678901, + "timestampNanosecond" : 0, + "type" : 1 + } + ], + "metadata":{ + "container.type":"docker_json-file" + } + })"; + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson.str()).c_str(), CompactJson(outJson).c_str()); + } } }