Add ProcessorParseContainerLogNative

core/models/PipelineEventGroup.cpp

@@ -16,6 +16,8 @@
 
 #include "models/PipelineEventGroup.h"
 
+#include <processor/ProcessorParseContainerLogNative.h>
+
 #include <sstream>
 
 #include "logger/Logger.h"
@@ -157,6 +159,7 @@ const std::string EVENT_GROUP_META_LOG_FILE_PATH_RESOLVED = "log.file.path_resol
 const std::string EVENT_GROUP_META_LOG_FILE_INODE = "log.file.inode";
 const std::string EVENT_GROUP_META_LOG_FILE_OFFSET = "log.file.offset";
 const std::string EVENT_GROUP_META_LOG_FILE_LENGTH = "log.file.length";
+const std::string EVENT_GROUP_META_CONTAINER_TYPE = "container.type";
 
 const std::string EVENT_GROUP_META_K8S_CLUSTER_ID = "k8s.cluster.id";
 const std::string EVENT_GROUP_META_K8S_NODE_NAME = "k8s.node.name";
@@ -169,6 +172,9 @@ const std::string EVENT_GROUP_META_CONTAINER_IP = "container.ip";
 const std::string EVENT_GROUP_META_CONTAINER_IMAGE_NAME = "container.image.name";
 const std::string EVENT_GROUP_META_CONTAINER_IMAGE_ID = "container.image.id";
 
+const std::string EVENT_GROUP_META_CONTAINERD_TEXT = "containerd_text";
+const std::string EVENT_GROUP_META_DOCKER_JSON_FILE = "docker_json-file";
+
 const std::string& EventGroupMetaKeyToString(EventGroupMetaKey key) {
     switch (key) {
         case EventGroupMetaKey::AGENT_TAG:
@@ -189,12 +195,23 @@ const std::string& EventGroupMetaKeyToString(EventGroupMetaKey key) {
             return EVENT_GROUP_META_LOG_FILE_OFFSET;
         case EventGroupMetaKey::LOG_READ_LENGTH:
             return EVENT_GROUP_META_LOG_FILE_LENGTH;
+        case EventGroupMetaKey::LOG_FORMAT:
+            return EVENT_GROUP_META_CONTAINER_TYPE;
         default:
             static std::string sEmpty = "unknown";
             return sEmpty;
     }
 }
 
+const std::string EventGroupMetaValueToString(std::string value) {
+    if (value == ProcessorParseContainerLogNative::CONTAINERD_TEXT) {
+        return EVENT_GROUP_META_CONTAINERD_TEXT;
+    } else if (value == ProcessorParseContainerLogNative::DOCKER_JSON_FILE) {
+        return EVENT_GROUP_META_DOCKER_JSON_FILE;
+    }
+    return value;
+}
+
 EventGroupMetaKey StringToEventGroupMetaKey(const std::string& key) {
     static std::unordered_map<std::string, EventGroupMetaKey> sStringToEnum
         = {{EVENT_GROUP_META_AGENT_TAG, EventGroupMetaKey::AGENT_TAG},
@@ -218,7 +235,7 @@ Json::Value PipelineEventGroup::ToJson() const {
     if (!mMetadata.empty()) {
         Json::Value metadata;
         for (const auto& meta : mMetadata) {
-            metadata[EventGroupMetaKeyToString(meta.first)] = meta.second.to_string();
+            metadata[EventGroupMetaKeyToString(meta.first)] = EventGroupMetaValueToString(meta.second.to_string());
         }
         root["metadata"] = metadata;
     }

core/models/PipelineEventGroup.h

@@ -42,6 +42,7 @@ enum class EventGroupMetaKey {
     LOG_FILE_INODE,
     LOG_READ_OFFSET,
     LOG_READ_LENGTH,
+    LOG_FORMAT,
 
     K8S_CLUSTER_ID,
     K8S_NODE_NAME,

core/monitor/MetricConstants.cpp

@@ -40,6 +40,7 @@ const std::string METRIC_PROC_PARSE_IN_SIZE_BYTES = "proc_parse_in_size_bytes";
 const std::string METRIC_PROC_PARSE_OUT_SIZE_BYTES = "proc_parse_out_size_bytes";
 
 const std::string METRIC_PROC_PARSE_ERROR_TOTAL = "proc_parse_error_total";
+const std::string METRIC_PROC_PARSE_SUCCESS_TOTAL = "proc_parse_success_total";
 const std::string METRIC_PROC_KEY_COUNT_NOT_MATCH_ERROR_TOTAL = "proc_key_count_not_match_error_total";
 const std::string METRIC_PROC_HISTORY_FAILURE_TOTAL = "proc_history_failure_total";
 
@@ -62,4 +63,8 @@ const std::string METRIC_PROC_MERGE_MULTILINE_LOG_MERGED_RECORDS_TOTAL
 const std::string METRIC_PROC_MERGE_MULTILINE_LOG_UNMATCHED_RECORDS_TOTAL
     = "proc_merge_multiline_log_unmatched_records_total";
 
+// processor parse container log native metrics
+const std::string METRIC_PROC_PARSE_STDOUT_TOTAL = "proc_parse_stdout_total";
+const std::string METRIC_PROC_PARSE_STDERR_TOTAL = "proc_parse_stderr_total";
+
 } // namespace logtail

core/monitor/MetricConstants.h

@@ -40,6 +40,7 @@ extern const std::string METRIC_PROC_TIME_MS;
 extern const std::string METRIC_PROC_PARSE_IN_SIZE_BYTES;
 extern const std::string METRIC_PROC_PARSE_OUT_SIZE_BYTES;
 extern const std::string METRIC_PROC_PARSE_ERROR_TOTAL;
+extern const std::string METRIC_PROC_PARSE_SUCCESS_TOTAL;
 extern const std::string METRIC_PROC_KEY_COUNT_NOT_MATCH_ERROR_TOTAL;
 extern const std::string METRIC_PROC_HISTORY_FAILURE_TOTAL;
 
@@ -56,4 +57,9 @@ extern const std::string METRIC_PROC_DESENSITIZE_RECORDS_TOTAL;
 extern const std::string METRIC_PROC_MERGE_MULTILINE_LOG_MERGED_RECORDS_TOTAL;
 extern const std::string METRIC_PROC_MERGE_MULTILINE_LOG_UNMATCHED_RECORDS_TOTAL;
 
+
+// processor parse container log native metrics
+extern const std::string METRIC_PROC_PARSE_STDOUT_TOTAL;
+extern const std::string METRIC_PROC_PARSE_STDERR_TOTAL;
+
 } // namespace logtail

core/plugin/PluginRegistry.cpp

@@ -42,6 +42,7 @@
 #include "processor/ProcessorFilterNative.h"
 #include "processor/ProcessorMergeMultilineLogNative.h"
 #include "processor/ProcessorParseApsaraNative.h"
+#include "processor/ProcessorParseContainerLogNative.h"
 #include "processor/ProcessorParseDelimiterNative.h"
 #include "processor/ProcessorParseJsonNative.h"
 #include "processor/ProcessorParseRegexNative.h"
@@ -261,6 +262,7 @@ void PluginRegistry::LoadStaticPlugins() {
     RegisterProcessorCreator(new StaticProcessorCreator<ProcessorTagNative>());
     RegisterProcessorCreator(new StaticProcessorCreator<ProcessorFilterNative>());
     RegisterProcessorCreator(new StaticProcessorCreator<ProcessorMergeMultilineLogNative>());
+    RegisterProcessorCreator(new StaticProcessorCreator<ProcessorParseContainerLogNative>());
 #if defined(__linux__) && !defined(__ANDROID__)
     if (BOOL_FLAG(enable_processor_spl)) {
         RegisterProcessorCreator(new StaticProcessorCreator<ProcessorSPL>());

core/processor/ProcessorMergeMultilineLogNative.h

@@ -62,6 +62,7 @@ class ProcessorMergeMultilineLogNative : public Processor {
     friend class ProcessorMergeMultilineLogNativeUnittest;
     friend class ProcessorMergeMultilineLogDisacardUnmatchUnittest;
     friend class ProcessorMergeMultilineLogKeepUnmatchUnittest;
+    friend class ProcessorParseContainerLogNativeUnittest;
 #endif
 };