Rework ring-buffer interface in prep of OMPT

Currently, lo2s creates on piece of shared memory (using shm_open) to
share one ring-buffer between lo2s and the child process.

Without the headache of synchronizing multiple event sources, this
ring-buffer can only be used by one event source, in this case it is
currently only used by CUPTI.

As a replacement, we offer a Unix Domain Socket (by default
/tmp/lo2s.socket). A source interested in submitting events to lo2s
creates a piece of shared memory and then sends the FD associated with
the Shared Memory to the lo2s socket.

lo2s then also maps the shared memory associated with that fd, and the
programs can communicate over that.

Every ring-buffer is read by its own RingbufMonitor thread.
This allows for an arbitrary amount of different measurement paradigms
in different threads
to connect themselves to lo2s via shared memory (one ringbuffer per OMPT thread, one
ringbuffer per CUDA process etc.)

.github/workflows/cmake.yml

@@ -38,3 +38,8 @@ jobs:
         with:
           overwrite: true
           path: lo2s*.tar.bz2
+      - name: Test
+        if: ${{ matrix.compiler == 'g++-12' && matrix.os == 'ubuntu-24.04' && matrix.build-type == 'RelWithDebInfo' && matrix.hw_breakpoint == 'ON' && github.event_name != 'pull_request'}}
+        run: |
+          sudo sysctl kernel.perf_event_paranoid=-1
+          ctest --output-on-failure

CMakeLists.txt

@@ -18,6 +18,9 @@ include(cmake/UnsetIfUpdated.cmake)
 include(cmake/CheckNameExists.cmake)
 include(cmake/CheckStructHasBitField.cmake)
 
+enable_testing()
+include(cmake/Tests.cmake)
+
 # Intialize git submodules if not done already
 include(cmake/GitSubmoduleUpdate.cmake)
 git_submodule_update()
@@ -182,6 +185,9 @@ set(SOURCE_FILES
 
     src/perf/event_resolver.cpp
     src/perf/event_attr.cpp
+    src/monitor/socket_monitor.cpp
+    src/monitor/ringbuf_monitor.cpp
+
 
     src/perf/bio/block_device.cpp
     src/perf/event_composer.cpp
@@ -325,6 +331,15 @@ if (USE_LIBAUDIT)
     endif()
 endif()
 
+add_executable(rb_test src/cupti/test.cpp)
+target_include_directories(rb_test PRIVATE include ${CMAKE_CURRENT_BINARY_DIR}/include)
+	target_link_libraries(rb_test PRIVATE fmt::fmt
+            Nitro::log
+            Nitro::env
+            Nitro::dl
+            Nitro::options
+	    otf2xx::Writer)
+
 set(LO2S_CUDA_INJECTIONLIB_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/liblo2s_injection.so")
 if(USE_CUPTI)
     if(CUDAToolkit_FOUND)

cmake/Tests.cmake

@@ -0,0 +1,7 @@
+set(TESTS lo2s_user_sampling)
+
+foreach(TEST ${TESTS})
+
+    add_test(NAME ${TEST} COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests/${TEST}.sh)
+    set_tests_properties(${TEST} PROPERTIES SKIP_RETURN_CODE 100)
+endforeach()

cmake/tests/lo2s_user_sampling.sh

@@ -0,0 +1,24 @@
+set -euo pipefail
+
+paranoid=$(cat /proc/sys/kernel/perf_event_paranoid)
+
+echo "perf_event_paranoid is: $paranoid"
+if [ $paranoid -gt 2 ]
+then
+    # skip
+    exit 100
+fi
+
+#!/bin/bash
+rm -rf test_trace
+./lo2s -c 1  --output-trace test_trace -- sleep 1
+
+if  otf2-print test_trace/traces.otf2 | grep  "SAMPLE"
+then
+    exit 0
+else
+    echo "Trace did not contain calling context samples!"
+    exit 1
+fi
+rm -rf test_trace
+

include/lo2s/config.hpp

@@ -105,6 +105,8 @@ struct Config
     bool use_nvidia = false;
     std::string cuda_injectionlib_path;
     uint64_t nvidia_ringbuf_size;
+
+    std::string socket_path;
 };
 
 const Config& config();

include/lo2s/cupti/events.hpp

@@ -22,11 +22,10 @@
 #pragma once
 
 #include <cstdint>
+#include <sys/types.h>
 
 namespace lo2s
 {
-namespace cupti
-{
 enum class EventType : uint64_t
 {
     CUPTI_KERNEL = 1,
@@ -46,5 +45,4 @@ struct event_kernel
     char name[1];
 };
 
-} // namespace cupti
 } // namespace lo2s

include/lo2s/monitor/abstract_process_monitor.hpp

@@ -41,7 +41,7 @@ class AbstractProcessMonitor
     virtual void insert_process(Process parent, Process process, std::string proc_name,
                                 bool spawn = false) = 0;
     virtual void insert_thread(Process process, Thread thread, std::string name = "",
-                               bool spawn = false, bool is_process = false) = 0;
+                               bool spawn = false) = 0;
 
     virtual void exit_thread(Thread thread) = 0;
 

include/lo2s/monitor/fwd.hpp

@@ -36,5 +36,7 @@ class CpuSwitchMonitor;
 class MainMonitor;
 class ProcessMonitor;
 class CpuSetMonitor;
+class SocketMonitor;
+class RingbufMonitor;
 } // namespace monitor
 } // namespace lo2s

include/lo2s/monitor/main_monitor.hpp

@@ -36,6 +36,7 @@
 #ifdef HAVE_VEOSINFO
 #include <lo2s/monitor/nec_monitor_main.hpp>
 #endif
+#include <lo2s/monitor/socket_monitor.hpp>
 #include <lo2s/monitor/tracepoint_monitor.hpp>
 #include <lo2s/process_info.hpp>
 #include <lo2s/trace/trace.hpp>
@@ -74,6 +75,7 @@ class MainMonitor
     metric::plugin::Metrics metrics_;
     std::vector<std::unique_ptr<TracepointMonitor>> tracepoint_monitors_;
 
+    std::unique_ptr<SocketMonitor> socket_monitor_;
     std::unique_ptr<IoMonitor<perf::bio::Writer>> bio_monitor_;
 #ifdef HAVE_X86_ADAPT
     std::unique_ptr<metric::x86_adapt::Metrics> x86_adapt_metrics_;

include/lo2s/monitor/process_monitor.hpp

@@ -45,8 +45,7 @@ class ProcessMonitor : public AbstractProcessMonitor, public MainMonitor
     ~ProcessMonitor();
     void insert_process(Process parent, Process child, std::string proc_name,
                         bool spawn = false) override;
-    void insert_thread(Process parent, Thread child, std::string name, bool spawn = false,
-                       bool is_process = false) override;
+    void insert_thread(Process parent, Thread child, std::string name, bool spawn = false) override;
 
     void exit_thread(Thread thread) override;
 

include/lo2s/monitor/ringbuf_monitor.hpp

@@ -0,0 +1,53 @@
+/*
+ * This file is part of the lo2s software.
+ * Linux OTF2 sampling
+ *
+ * Copyright (c) 2016-2018, Technische Universitaet Dresden, Germany
+ *
+ * lo2s is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * lo2s is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with lo2s.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <lo2s/monitor/fwd.hpp>
+#include <lo2s/monitor/main_monitor.hpp>
+#include <lo2s/monitor/poll_monitor.hpp>
+#include <lo2s/ringbuf.hpp>
+
+namespace lo2s
+{
+namespace monitor
+{
+
+class RingbufMonitor : public PollMonitor
+{
+public:
+    RingbufMonitor(trace::Trace& trace, int fd);
+
+    void initialize_thread() override;
+    void finalize_thread() override;
+    void monitor(int fd) override;
+
+    std::string group() const override
+    {
+        return "lo2s::RingbufMonitor";
+    }
+
+private:
+    trace::Trace& trace_;
+    perf::time::Converter& time_converter_;
+    RingBufReader ringbuf_reader_;
+};
+} // namespace monitor
+} // namespace lo2s

include/lo2s/monitor/scope_monitor.hpp

@@ -24,7 +24,6 @@
 #include <lo2s/monitor/main_monitor.hpp>
 #include <lo2s/monitor/poll_monitor.hpp>
 
-#include <lo2s/cupti/reader.hpp>
 #include <lo2s/perf/counter/group/writer.hpp>
 #include <lo2s/perf/counter/userspace/writer.hpp>
 #include <lo2s/perf/sample/writer.hpp>
@@ -51,8 +50,7 @@ namespace monitor
 class ScopeMonitor : public PollMonitor
 {
 public:
-    ScopeMonitor(ExecutionScope scope, MainMonitor& parent, bool enable_on_exec,
-                 bool is_process = false);
+    ScopeMonitor(ExecutionScope scope, MainMonitor& parent, bool enable_on_exec);
 
     void initialize_thread() override;
     void finalize_thread() override;
@@ -76,7 +74,6 @@ class ScopeMonitor : public PollMonitor
     std::unique_ptr<perf::sample::Writer> sample_writer_;
     std::unique_ptr<perf::counter::group::Writer> group_counter_writer_;
     std::unique_ptr<perf::counter::userspace::Writer> userspace_counter_writer_;
-    std::unique_ptr<cupti::Reader> cupti_reader_;
 };
 } // namespace monitor
 } // namespace lo2s

include/lo2s/monitor/socket_monitor.hpp

@@ -0,0 +1,60 @@
+/*
+ * This file is part of the lo2s software.
+ * Linux OTF2 sampling
+ *
+ * Copyright (c) 2016-2018, Technische Universitaet Dresden, Germany
+ *
+ * lo2s is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * lo2s is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with lo2s.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <lo2s/monitor/fwd.hpp>
+#include <lo2s/monitor/main_monitor.hpp>
+#include <lo2s/monitor/poll_monitor.hpp>
+
+#include <lo2s/monitor/ringbuf_monitor.hpp>
+
+extern "C"
+{
+#include <sched.h>
+#include <unistd.h>
+}
+
+namespace lo2s
+{
+namespace monitor
+{
+
+class SocketMonitor : public PollMonitor
+{
+public:
+    SocketMonitor(trace::Trace& trace);
+
+    void initialize_thread() override;
+    void finalize_thread() override;
+    void monitor(int fd) override;
+
+    std::string group() const override
+    {
+        return "lo2s::SocketMonitor";
+    }
+
+private:
+    trace::Trace& trace_;
+    std::map<int, RingbufMonitor> monitors_;
+    int socket = -1;
+};
+} // namespace monitor
+} // namespace lo2s

include/lo2s/monitor/system_process_monitor.hpp

@@ -46,8 +46,8 @@ class SystemProcessMonitor : public AbstractProcessMonitor
     virtual void insert_process(Process parent, Process process, std::string proc_name,
                                 bool spawn) override;
 
-    virtual void insert_thread(Process process, Thread thread, std::string name, bool spawn,
-                               bool is_process) override;
+    virtual void insert_thread(Process process, Thread thread, std::string name,
+                               bool spawn) override;
 
     virtual void exit_thread(Thread thread) override;