feat: support host monitor

core/CMakeLists.txt

@@ -130,6 +130,7 @@ set(SUB_DIRECTORIES_LIST
         prometheus prometheus/labels prometheus/schedulers prometheus/async
         ebpf ebpf/observer ebpf/security ebpf/handler
         parser sls_control sdk
+        host_monitor host_monitor/collector
         )
 if (LINUX)
     if (ENABLE_ENTERPRISE)

core/app_config/AppConfig.cpp

@@ -166,6 +166,7 @@ DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator s
 DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
 DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
 DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");
+DEFINE_FLAG_INT32(process_collect_silent_count, "number of process scanned between a sleep", 1000);
 
 DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
 DEFINE_FLAG_STRING(crash_stack_file_name, "crash stack back trace file name", "backtrace.dat");

core/application/Application.cpp

@@ -14,6 +14,8 @@
 
 #include "application/Application.h"
 
+#include "timer/Timer.h"
+
 #ifndef LOGTAIL_NO_TC_MALLOC
 #include <gperftools/malloc_extension.h>
 #endif
@@ -259,9 +261,9 @@ void Application::Start() { // GCOVR_EXCL_START
     }
 
     ProcessorRunner::GetInstance()->Init();
+    Timer::GetInstance()->Init();
 
-    time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0,
-           lastCheckTagsTime = 0, lastQueueGCTime = 0;
+    time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, lastCheckTagsTime = 0, lastQueueGCTime = 0;
 #ifndef LOGTAIL_NO_TC_MALLOC
     time_t lastTcmallocReleaseMemTime = 0;
 #endif

core/common/FileSystemUtil.cpp

@@ -141,6 +141,50 @@ bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t
     return true;
 }
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    std::string line;
+    // 此处必须判断eof，具体原因参见:
+    // https://stackoverflow.com/questions/40561482/getline-throws-basic-iosclear-exception-after-reading-the-last-line
+    while (!is.eof() && std::getline(is, line)) {
+        if (enableEmptyLine || !line.empty()) {
+            pushBack(line);
+        }
+    }
+    return 0;
+}
+
+int GetLines(const std::filesystem::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage) {
+    int ret = 0;
+    std::ifstream fin;
+    try {
+        fin.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+        fin.open(filename.string(), std::ios_base::in);
+        fin.exceptions(std::ifstream::goodbit);
+        GetLines(fin, enableEmptyLine, pushBack, errorMessage);
+        fin.close();
+    } catch (const std::exception& fail) {
+        if (errorMessage != nullptr) {
+            LOG_ERROR(sLogger, ("open file fail", filename)("errno", strerror(errno)));
+            ret = -1;
+        }
+        fin.close();
+    }
+    return ret;
+}
+
+int GetFileLines(const std::filesystem::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine,
+                 std::string* errorMessage) {
+    return GetLines(filename, enableEmptyLine, [&res](const std::string& s) { res.push_back(s); }, errorMessage);
+}
+
 bool OverwriteFile(const std::string& fileName, const std::string& content) {
     FILE* pFile = fopen(fileName.c_str(), "w");
     if (pFile == NULL) {

core/common/FileSystemUtil.h

@@ -26,6 +26,8 @@
 #elif defined(_MSC_VER)
 #include <Windows.h>
 #endif
+#include <filesystem>
+
 #include "DevInode.h"
 #include "ErrorUtil.h"
 #include "LogtailCommonFlags.h"
@@ -87,6 +89,19 @@ void TrimLastSeperator(std::string& path);
 // ReadFileContent reads all content of @fileName to @content.
 bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t maxFileSize = 8192);
 
+int GetLines(std::istream& is,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetLines(const std::filesystem::path& filename,
+             bool enableEmptyLine,
+             const std::function<void(const std::string&)>& pushBack,
+             std::string* errorMessage);
+int GetFileLines(const std::filesystem::path& filename,
+                 std::vector<std::string>& res,
+                 bool enableEmptyLine = true,
+                 std::string* errorMessage = nullptr);
+
 // OverwriteFile overwrides @fileName with @content.
 bool OverwriteFile(const std::string& fileName, const std::string& content);
 

core/common/MachineInfoUtil.cpp

@@ -15,6 +15,7 @@
 #include "MachineInfoUtil.h"
 
 #include <string.h>
+
 #if defined(__linux__)
 #include <arpa/inet.h>
 #include <ifaddrs.h>
@@ -38,8 +39,8 @@
 
 #include <thread>
 
-#include "FileSystemUtil.h"
 #include "StringTools.h"
+#include "common/FileSystemUtil.h"
 #include "logger/Logger.h"
 
 

core/common/MachineInfoUtil.h

@@ -15,10 +15,11 @@
  */
 
 #pragma once
+#include <rapidjson/document.h>
+#include <rapidjson/rapidjson.h>
+
 #include <string>
 #include <unordered_set>
-#include <rapidjson/rapidjson.h>
-#include <rapidjson/document.h>
 
 namespace logtail {
 
@@ -37,7 +38,7 @@ std::string GetHostIp(const std::string& intf = "");
 void GetAllPids(std::unordered_set<int32_t>& pids);
 bool GetKernelInfo(std::string& kernelRelease, int64_t& kernelVersion);
 bool GetRedHatReleaseInfo(std::string& os, int64_t& osVersion, std::string bashPath = "");
-bool IsDigitsDotsHostname(const char *hostname);
+bool IsDigitsDotsHostname(const char* hostname);
 ECSMeta FetchECSMeta();
 
 // GetAnyAvailableIP walks through all interfaces (AF_INET) to find an available IP.

core/common/StringTools.cpp

@@ -363,4 +363,12 @@ void RemoveFilePathTrailingSlash(std::string& filePath) {
     filePath = path.string();
 }
 
+bool IsInt(const char* sz) {
+    bool ok = (sz != nullptr && *sz != '\0');
+    for (auto* it = sz; ok && *it; ++it) {
+        ok = (0 != std::isdigit(*it));
+    }
+    return ok;
+}
+
 } // namespace logtail

core/common/StringTools.h

@@ -146,6 +146,12 @@ bool NormalizeTopicRegFormat(std::string& regStr);
 
 void RemoveFilePathTrailingSlash(std::string& path);
 
+bool IsInt(const char* sz);
+
+inline bool IsInt(const std::string& str) {
+    return IsInt(str.c_str());
+}
+
 #if defined(_MSC_VER)
 // TODO: Test it.
 #define FNM_PATHNAME 0

core/common/ThreadPool.h

@@ -15,13 +15,13 @@
  */
 
 #pragma once
-#include <thread>
-#include <mutex>
 #include <atomic>
 #include <condition_variable>
 #include <functional>
-#include <vector>
+#include <mutex>
 #include <queue>
+#include <thread>
+#include <vector>
 
 namespace logtail {
 

core/common/timer/Timer.cpp

@@ -23,6 +23,9 @@ namespace logtail {
 void Timer::Init() {
     {
         lock_guard<mutex> lock(mThreadRunningMux);
+        if (mIsThreadRunning) {
+            return;
+        }
         mIsThreadRunning = true;
     }
     mThreadRes = async(launch::async, &Timer::Run, this);
@@ -31,6 +34,9 @@ void Timer::Init() {
 void Timer::Stop() {
     {
         lock_guard<mutex> lock(mThreadRunningMux);
+        if (!mIsThreadRunning) {
+            return;
+        }
         mIsThreadRunning = false;
     }
     mCV.notify_one();
@@ -45,6 +51,13 @@ void Timer::Stop() {
     }
 }
 
+void Timer::Clear() {
+    lock_guard<mutex> lock(mQueueMux);
+    while (!mQueue.empty()) {
+        mQueue.pop();
+    }
+}
+
 void Timer::PushEvent(unique_ptr<TimerEvent>&& e) {
     lock_guard<mutex> lock(mQueueMux);
     if (mQueue.empty() || e->GetExecTime() < mQueue.top()->GetExecTime()) {

core/common/timer/Timer.h

@@ -34,11 +34,22 @@ struct TimerEventCompare {
 
 class Timer {
 public:
+    Timer(const Timer&) = delete;
+    Timer(Timer&&) = delete;
+    Timer& operator=(const Timer&) = delete;
+    Timer& operator=(Timer&&) = delete;
+    ~Timer() = default;
+    static Timer* GetInstance() {
+        static Timer sInstance;
+        return &sInstance;
+    }
     void Init();
     void Stop();
     void PushEvent(std::unique_ptr<TimerEvent>&& e);
+    void Clear();
 
 private:
+    Timer() = default;
     void Run();
 
     mutable std::mutex mQueueMux;
@@ -47,12 +58,13 @@ class Timer {
 
     std::future<void> mThreadRes;
     mutable std::mutex mThreadRunningMux;
-    bool mIsThreadRunning = true;
+    bool mIsThreadRunning = false;
     mutable std::condition_variable mCV;
 
 #ifdef APSARA_UNIT_TEST_MAIN
     friend class TimerUnittest;
     friend class ScrapeSchedulerUnittest;
+    friend class HostMonitorInputRunnerUnittest;
 #endif
 };
 

core/common/timer/TimerEvent.h

@@ -29,6 +29,7 @@ class TimerEvent {
     virtual bool Execute() = 0;
 
     std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; }
+    void SetExecTime(std::chrono::steady_clock::time_point nextExecTime) { mExecTime = nextExecTime; }
 
 private:
     std::chrono::steady_clock::time_point mExecTime;

core/constants/EntityConstants.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "EntityConstants.h"
+
+namespace logtail {
+
+const std::string DEFAULT_HOST_TYPE_ECS = "ecs";
+const std::string DEFAULT_HOST_TYPE_HOST = "host";
+const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE = "__entity_type__";
+const std::string DEFAULT_CONTENT_KEY_ENTITY_ID = "__entity_id__";
+const std::string DEFAULT_CONTENT_KEY_DOMAIN = "__domain__";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS = "acs";
+const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA = "infra";
+const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME = "__first_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME = "__last_observed_time__";
+const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS = "__keep_alive_seconds__";
+const std::string DEFAULT_CONTENT_KEY_METHOD = "__method__";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE = "update";
+const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE = "expire";
+
+// for process entity
+const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS = "process";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PID = "pid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID = "ppid";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_USER = "user";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM = "comm";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME = "create_time";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD = "cwd";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY = "binary";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS = "arguments";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE = "language";
+const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID = "container_id";
+} // namespace logtail

core/constants/EntityConstants.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+
+namespace logtail {
+
+extern const std::string DEFAULT_ENV_KEY_HOST_TYPE;
+extern const std::string DEFAULT_HOST_TYPE_ECS;
+extern const std::string DEFAULT_HOST_TYPE_HOST;
+extern const std::string DEFAULT_CONTENT_KEY_ENTITY_TYPE;
+extern const std::string DEFAULT_CONTENT_KEY_ENTITY_ID;
+extern const std::string DEFAULT_CONTENT_KEY_DOMAIN;
+extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_ACS;
+extern const std::string DEFAULT_CONTENT_VALUE_DOMAIN_INFRA;
+extern const std::string DEFAULT_CONTENT_KEY_FIRST_OBSERVED_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_LAST_OBSERVED_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_KEEP_ALIVE_SECONDS;
+extern const std::string DEFAULT_CONTENT_KEY_METHOD;
+extern const std::string DEFAULT_CONTENT_VALUE_METHOD_UPDATE;
+extern const std::string DEFAULT_CONTENT_VALUE_METHOD_EXPIRE;
+
+// for process entity
+extern const std::string DEFAULT_CONTENT_VALUE_ENTITY_TYPE_PROCESS;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PID;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_PPID;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_USER;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_COMM;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CREATE_TIME;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CWD;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_BINARY;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_ARGUMENTS;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_LANGUAGE;
+extern const std::string DEFAULT_CONTENT_KEY_PROCESS_CONTAINER_ID;
+} // namespace logtail