From 50ddcd32eb217e1aa5e87124883ee284a36052a1 Mon Sep 17 00:00:00 2001 From: Dom Delnano Date: Mon, 29 Jul 2024 02:10:14 -0400 Subject: [PATCH] Add `--stirling_uprobe_opt_out` cli flag to allow opting binaries from uprobe attachment (#1971) Summary: Add `--stirling_uprobe_opt_out` cli flag to allow opting binaries from uprobe attachment See https://github.com/pixie-io/pixie/issues/1970 for the motivation of this change. Relevant Issues: https://github.com/pixie-io/pixie/issues/1970 Type of change: /kind feature Test Plan: Verified the following scenarios - [x] BPF tests pass on all kernels -- see [this build-and-test run](https://github.com/pixie-io/pixie/actions/runs/10063985263) for the results. - [x] Dynamically linked OpenSSL application is skipped from uprobe attachment ``` # Run //src/stirling/source_connectors/socket_tracer/testing/containers:nginx_alpine_openssl_3_0_8_image # Run stirling_wrapper with --stirling_uprobe_opt_out=nginx,non_matching --vmodule=uprobe_manager=1 [ .. ] I20240723 17:40:54.208112 739524 uprobe_manager.cc:584] binary filename '/proc/738767/root/usr/sbin/nginx' contained in uprobe opt out list, skipping. I20240723 17:40:54.209451 739524 uprobe_manager.cc:584] binary filename '/proc/738731/root/usr/sbin/nginx' contained in uprobe opt out list, skipping. ``` - [x] Statically linked BoringSSL application is skipped from uprobe attachment ``` # Run //src/stirling/source_connectors/socket_tracer/testing/containers/bssl:bssl_image # Run stirling_wrapper with --stirling_uprobe_opt_out=bssl,non_matching --vmodule=uprobe_manager=1 [ .. ] I20240723 17:51:14.265595 742187 uprobe_manager.cc:584] binary filename '/proc/741971/root/app/bssl.runfiles/boringssl/bssl' contained in uprobe opt out list, skipping. ``` - [x] Nodejs application is skipped from uprobe attachment ``` # Run //src/stirling/source_connectors/socket_tracer/testing/containers:node_14_18_1_alpine_image # Run stirling_wrapper with --stirling_uprobe_opt_out=https_server.js,non_matching --vmodule=uprobe_manager=1 [ .. ] I20240723 16:50:01.197883 725584 uprobe_manager.cc:486] binary filename 'https_server.js' contained in uprobe opt out list, skipping. ``` - [x] Go application attachment is skipped with `--stirling_uprobe_opt_out=https_server.js,non_matching` ``` # Run //src/stirling/testing/demo_apps/go_https/server:golang_1_21_https_server # Run stirling_wrapper with --stirling_uprobe_opt_out=golang_1_21_server_binary,non_matching --vmodule=uprobe_manager=1 [ .. ] I20240723 17:29:54.166582 736461 uprobe_manager.cc:635] binary filename '/golang_1_21_server_binary' contained in uprobe opt out list, skipping. ``` Changelog Message: Add mechanism for opting out specific binaries from uprobe attachment --------- Signed-off-by: Dom Del Nano --- .../socket_tracer/uprobe_manager.cc | 43 ++++++++++++++++--- .../socket_tracer/uprobe_manager.h | 6 ++- src/stirling/utils/detect_application.cc | 17 ++++++++ src/stirling/utils/detect_application.h | 3 ++ src/stirling/utils/detect_application_test.cc | 6 +++ 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/stirling/source_connectors/socket_tracer/uprobe_manager.cc b/src/stirling/source_connectors/socket_tracer/uprobe_manager.cc index 25ac91fe315..f79dbfa6fa5 100644 --- a/src/stirling/source_connectors/socket_tracer/uprobe_manager.cc +++ b/src/stirling/source_connectors/socket_tracer/uprobe_manager.cc @@ -52,6 +52,11 @@ DEFINE_double(stirling_rescan_exp_backoff_factor, 2.0, "Exponential backoff factor used in decided how often to rescan binaries for " "dynamically loaded libraries"); +DEFINE_string( + stirling_uprobe_opt_out, "", + "Comma separated list of binary filenames that should be excluded from uprobe attachment." + "For a binary at path /path/to/binary, the filename would be binary"); + namespace px { namespace stirling { @@ -62,8 +67,13 @@ using ::px::system::KernelVersion; using ::px::system::KernelVersionOrder; using ::px::system::ProcPidRootPath; +constexpr std::string_view kUprobeSkippedMessage = + "binary filename '$0' contained in uprobe opt out list, skipping."; + UProbeManager::UProbeManager(bpf_tools::BCCWrapper* bcc) : bcc_(bcc) { proc_parser_ = std::make_unique(); + auto opt_out_list = absl::StrSplit(FLAGS_stirling_uprobe_opt_out, ",", absl::SkipWhitespace()); + uprobe_opt_out_ = absl::flat_hash_set(opt_out_list.begin(), opt_out_list.end()); } void UProbeManager::Init(bool disable_go_tls_tracing, bool enable_http2_tracing, @@ -447,8 +457,8 @@ StatusOr> UProbeManager::GetNodeOpensslUProbeTmpls(con return iter->second; } -StatusOr UProbeManager::AttachOpenSSLUProbesOnStaticBinary(const uint32_t pid) { - PX_ASSIGN_OR_RETURN(const std::filesystem::path proc_exe, proc_parser_->GetExePath(pid)); +StatusOr UProbeManager::AttachOpenSSLUProbesOnStaticBinary( + const uint32_t pid, const std::filesystem::path& proc_exe) { const auto host_proc_exe = ProcPidRootPath(pid, proc_exe); PX_ASSIGN_OR_RETURN(auto elf_reader, ElfReader::Create(host_proc_exe)); @@ -467,13 +477,19 @@ StatusOr UProbeManager::AttachOpenSSLUProbesOnStaticBinary(const uint32_t p return kOpenSSLUProbes.size(); } -StatusOr UProbeManager::AttachNodeJsOpenSSLUprobes(const uint32_t pid) { - PX_ASSIGN_OR_RETURN(const std::filesystem::path proc_exe, proc_parser_->GetExePath(pid)); - +StatusOr UProbeManager::AttachNodeJsOpenSSLUprobes(const uint32_t pid, + const std::filesystem::path& proc_exe) { if (DetectApplication(proc_exe) != Application::kNode) { return 0; } + const std::string exe_cmdline = proc_parser_->GetPIDCmdline(pid); + const auto node_application_filepath = GetNodeApplicationFilename(exe_cmdline); + if (node_application_filepath.has_value() && + uprobe_opt_out_.contains(node_application_filepath.value())) { + VLOG(1) << absl::Substitute(kUprobeSkippedMessage, node_application_filepath.value()); + return 0; + } const auto host_proc_exe = ProcPidRootPath(pid, proc_exe); const auto [_, inserted] = nodejs_binaries_.insert(host_proc_exe.string()); @@ -608,6 +624,13 @@ int UProbeManager::DeployOpenSSLUProbes(const absl::flat_hash_set& pid continue; } + PX_ASSIGN_OR(const auto exe_path, proc_parser_->GetExePath(pid.pid()), continue); + + if (uprobe_opt_out_.contains(exe_path.filename().string())) { + VLOG(1) << absl::Substitute(kUprobeSkippedMessage, exe_path.string()); + continue; + } + auto count_or = AttachOpenSSLUProbesOnDynamicLib(pid.pid()); if (count_or.ok()) { uprobe_count += count_or.ValueOrDie(); @@ -622,7 +645,7 @@ int UProbeManager::DeployOpenSSLUProbes(const absl::flat_hash_set& pid count_or.ToString()); } - count_or = AttachNodeJsOpenSSLUprobes(pid.pid()); + count_or = AttachNodeJsOpenSSLUprobes(pid.pid(), exe_path); if (count_or.ok()) { uprobe_count += count_or.ValueOrDie(); VLOG(1) << absl::Substitute( @@ -640,7 +663,7 @@ int UProbeManager::DeployOpenSSLUProbes(const absl::flat_hash_set& pid // Attach uprobes to statically linked applications only if no other probes have been attached. if (FLAGS_stirling_trace_static_tls_binaries && count_or.ok() && count_or.ValueOrDie() == 0) { - count_or = AttachOpenSSLUProbesOnStaticBinary(pid.pid()); + count_or = AttachOpenSSLUProbesOnStaticBinary(pid.pid(), exe_path); if (count_or.ok() && count_or.ValueOrDie() > 0) { uprobe_count += count_or.ValueOrDie(); @@ -817,6 +840,12 @@ int UProbeManager::DeployGoUProbes(const absl::flat_hash_set& pids) { static int32_t kPID = getpid(); for (const auto& [binary, pid_vec] : ConvertPIDsListToMap(pids)) { + std::filesystem::path binary_path(binary); + auto binary_filepath = binary_path.filename().string(); + if (uprobe_opt_out_.contains(binary_filepath)) { + VLOG(1) << absl::Substitute(kUprobeSkippedMessage, binary_filepath); + continue; + } // Don't bother rescanning binaries that have been scanned before to avoid unnecessary work. if (!scanned_binaries_.insert(binary).second) { continue; diff --git a/src/stirling/source_connectors/socket_tracer/uprobe_manager.h b/src/stirling/source_connectors/socket_tracer/uprobe_manager.h index 82f5660c27d..b3f8fc8b25b 100644 --- a/src/stirling/source_connectors/socket_tracer/uprobe_manager.h +++ b/src/stirling/source_connectors/socket_tracer/uprobe_manager.h @@ -538,7 +538,7 @@ class UProbeManager { * @return The number of uprobes deployed. It is not an error if the binary * does not use OpenSSL; instead the return value will be zero. */ - StatusOr AttachNodeJsOpenSSLUprobes(uint32_t pid); + StatusOr AttachNodeJsOpenSSLUprobes(uint32_t pid, const std::filesystem::path& binary_path); /** * Attaches the required probes for TLS tracing to the specified PID if the binary is @@ -551,7 +551,8 @@ class UProbeManager { * @return The number of uprobes deployed. It is not an error if the binary * does not contain the necessary symbols to probe; instead the return value will be zero. */ - StatusOr AttachOpenSSLUProbesOnStaticBinary(uint32_t pid); + StatusOr AttachOpenSSLUProbesOnStaticBinary(uint32_t pid, + const std::filesystem::path& binary_path); /** * Calls BCCWrapper.AttachUProbe() with a probe template and log any errors to the probe status @@ -628,6 +629,7 @@ class UProbeManager { // Without clean-up, these could consume more-and-more memory. absl::flat_hash_set openssl_probed_binaries_; absl::flat_hash_set scanned_binaries_; + absl::flat_hash_set uprobe_opt_out_; absl::flat_hash_set go_probed_binaries_; absl::flat_hash_set go_http2_probed_binaries_; absl::flat_hash_set go_tls_probed_binaries_; diff --git a/src/stirling/utils/detect_application.cc b/src/stirling/utils/detect_application.cc index 41ed89b4a24..548e71d8d6f 100644 --- a/src/stirling/utils/detect_application.cc +++ b/src/stirling/utils/detect_application.cc @@ -49,6 +49,23 @@ Application DetectApplication(const std::filesystem::path& exe) { return Application::kUnknown; } +// This method returns the main nodejs application file from a command line. See the following +// examples below: +// +// "node /usr/bin/test.js" -> "test.js" +// "node --node-memory-debug /usr/bin/test.js" -> "test.js" +// "node /usr/bin/test" -> std::nullopt +std::optional GetNodeApplicationFilename(std::string_view cmdline) { + std::vector cmdline_parts = absl::StrSplit(cmdline, ' '); + for (const auto& part : cmdline_parts) { + if (absl::EndsWith(part, ".js")) { + std::filesystem::path path(part); + return path.filename(); + } + } + return {}; +} + bool operator<(const SemVer& lhs, const SemVer& rhs) { std::vector lhs_vec = {lhs.major, lhs.minor, lhs.patch}; std::vector rhs_vec = {rhs.major, rhs.minor, rhs.patch}; diff --git a/src/stirling/utils/detect_application.h b/src/stirling/utils/detect_application.h index 3a0efe3a628..0e13d48535b 100644 --- a/src/stirling/utils/detect_application.h +++ b/src/stirling/utils/detect_application.h @@ -36,6 +36,9 @@ enum class Application { // Returns the application of the input executable. Application DetectApplication(const std::filesystem::path& exe); +// Returns the filename of a node application from the command line. +std::optional GetNodeApplicationFilename(std::string_view cmdline); + // Describes a semantic versioning number. struct SemVer { int major = 0; diff --git a/src/stirling/utils/detect_application_test.cc b/src/stirling/utils/detect_application_test.cc index 8b417e43e60..7cd1dc09853 100644 --- a/src/stirling/utils/detect_application_test.cc +++ b/src/stirling/utils/detect_application_test.cc @@ -34,6 +34,12 @@ TEST(DetectApplicationTest, ResultsAreAsExpected) { EXPECT_EQ(Application::kNode, DetectApplication("/usr/bin/nodejs")); } +TEST(GetNodeApplicatFilenameTest, ResultsAreAsExpected) { + EXPECT_EQ(GetNodeApplicationFilename("node /usr/bin/test.js"), "test.js"); + EXPECT_EQ(GetNodeApplicationFilename("node --node-memory-debug /usr/bin/test.js"), "test.js"); + EXPECT_FALSE(GetNodeApplicationFilename("node /usr/bin/test").has_value()); +} + TEST(GetSemVerTest, AsExpected) { ASSERT_OK_AND_ASSIGN(SemVer sem_ver, GetSemVer("v1.12.13-test", true)); EXPECT_EQ(sem_ver.major, 1);