Skip to content

Commit

Permalink
[SYCL][XPTI] Add unique queue ID to trace (intel#11548)
Browse files Browse the repository at this point in the history 
Signed-off-by: Tikhomirova, Kseniya <[email protected]>
  • Loading branch information
@KseniyaTikhomirova
KseniyaTikhomirova authored Oct 26, 2023
1 parent c4e8e2a commit da4b83c
Show file tree
Hide file tree
Showing 21 changed files with 676 additions and 114 deletions.
6 changes: 3 additions & 3 deletions sycl/doc/design/SYCLInstrumentationUsingXPTI.md
View file

Large diffs are not rendered by default.

7 changes: 6 additions & 1 deletion sycl/source/detail/queue_impl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
namespace sycl {
inline namespace _V1 {
namespace detail {

std::atomic<unsigned long long> queue_impl::MNextAvailableQueueID = 0;

template <>
uint32_t queue_impl::get_info<info::queue::reference_count>() const {
sycl::detail::pi::PiResult result = PI_SUCCESS;
Expand Down Expand Up @@ -75,6 +78,7 @@ event queue_impl::memset(const std::shared_ptr<detail::queue_impl> &Self,
xpti::addMetadata(TEvent, "memory_ptr", reinterpret_cast<size_t>(Ptr));
xpti::addMetadata(TEvent, "value_set", Value);
xpti::addMetadata(TEvent, "memory_size", Count);
xpti::addMetadata(TEvent, "queue_id", MQueueID);
});
// Notify XPTI about the memset submission
PrepareNotify.notify();
Expand Down Expand Up @@ -152,6 +156,7 @@ event queue_impl::memcpy(const std::shared_ptr<detail::queue_impl> &Self,
xpti::addMetadata(TEvent, "dest_memory_ptr",
reinterpret_cast<size_t>(Dest));
xpti::addMetadata(TEvent, "memory_size", Count);
xpti::addMetadata(TEvent, "queue_id", MQueueID);
});
// Notify XPTI about the memset submission
PrepareNotify.notify();
Expand Down Expand Up @@ -448,7 +453,7 @@ void *queue_impl::instrumentationProlog(const detail::code_location &CodeLoc,
DevStr = "ACCELERATOR";
else
DevStr = "UNKNOWN";
xpti::addMetadata(WaitEvent, "sycl_device", DevStr);
xpti::addMetadata(WaitEvent, "sycl_device_type", DevStr);
if (HasSourceInfo) {
xpti::addMetadata(WaitEvent, "sym_function_name", CodeLoc.functionName());
xpti::addMetadata(WaitEvent, "sym_source_file_name", CodeLoc.fileName());
Expand Down
105 changes: 59 additions & 46 deletions sycl/source/detail/queue_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,39 +113,9 @@ class queue_impl {
has_property<ext::oneapi::property::queue::discard_events>()),
MIsProfilingEnabled(has_property<property::queue::enable_profiling>()),
MHasDiscardEventsSupport(MDiscardEvents &&
(MHostQueue ? true : MIsInorder)) {
// We enable XPTI tracing events using the TLS mechanism; if the code
// location data is available, then the tracing data will be rich.
#if XPTI_ENABLE_INSTRUMENTATION
/// This section of code is relying on scoped objects, so they cannot be
/// encapsulated in a function
constexpr uint16_t NotificationTraceType =
static_cast<uint16_t>(xpti::trace_point_type_t::queue_create);
XPTIScope PrepareNotify((void *)this, NotificationTraceType,
SYCL_STREAM_NAME, "queue_create");
// Cache the trace event, stream id and instance IDs for the destructor
if (xptiCheckTraceEnabled(PrepareNotify.streamID(),
NotificationTraceType)) {
MTraceEvent = (void *)PrepareNotify.traceEvent();
MStreamID = PrepareNotify.streamID();
MInstanceID = PrepareNotify.instanceID();
// Add the function to capture meta data for the XPTI trace event
PrepareNotify.addMetadata([&](auto TEvent) {
xpti::addMetadata(TEvent, "sycl_context",
reinterpret_cast<size_t>(MContext->getHandleRef()));
if (MDevice) {
xpti::addMetadata(TEvent, "sycl_device_name",
MDevice->getDeviceName());
xpti::addMetadata(
TEvent, "sycl_device",
reinterpret_cast<size_t>(
MDevice->is_host() ? 0 : MDevice->getHandleRef()));
}
xpti::addMetadata(TEvent, "is_inorder", MIsInorder);
});
PrepareNotify.notify();
}
#endif
(MHostQueue ? true : MIsInorder)),
MQueueID{
MNextAvailableQueueID.fetch_add(1, std::memory_order_relaxed)} {
if (has_property<property::queue::enable_profiling>()) {
if (has_property<ext::oneapi::property::queue::discard_events>())
throw sycl::exception(make_error_code(errc::invalid),
Expand Down Expand Up @@ -196,28 +166,19 @@ class queue_impl {
// This section is the second part of the instrumentation that uses the
// tracepoint information and notifies
}
}

private:
void queue_impl_interop(sycl::detail::pi::PiQueue PiQueue) {
// The following commented section provides a guideline on how to use the
// TLS enabled mechanism to create a tracepoint and notify using XPTI. This
// is the prolog section and the epilog section will initiate the
// notification.
// We enable XPTI tracing events using the TLS mechanism; if the code
// location data is available, then the tracing data will be rich.
#if XPTI_ENABLE_INSTRUMENTATION
/// This section of code is relying on scoped objects, so they cannot be
/// encapsulated in a function
constexpr uint16_t NotificationTraceType =
static_cast<uint16_t>(xpti::trace_point_type_t::queue_create);
XPTIScope PrepareNotify((void *)this, NotificationTraceType,
SYCL_STREAM_NAME, "queue_create");
// Cache the trace event, stream id and instance IDs for the destructor
if (xptiCheckTraceEnabled(PrepareNotify.streamID(),
NotificationTraceType)) {
// Cache the trace event, stream id and instance IDs for the destructor
MTraceEvent = (void *)PrepareNotify.traceEvent();
MStreamID = PrepareNotify.streamID();
MInstanceID = PrepareNotify.instanceID();

// Add the function to capture meta data for the XPTI trace event
PrepareNotify.addMetadata([&](auto TEvent) {
xpti::addMetadata(TEvent, "sycl_context",
Expand All @@ -231,10 +192,18 @@ class queue_impl {
MDevice->is_host() ? 0 : MDevice->getHandleRef()));
}
xpti::addMetadata(TEvent, "is_inorder", MIsInorder);
xpti::addMetadata(TEvent, "queue_id", MQueueID);
if (!MHostQueue)
xpti::addMetadata(TEvent, "queue_handle",
reinterpret_cast<size_t>(getHandleRef()));
});
PrepareNotify.notify();
}
#endif
}

private:
void queue_impl_interop(sycl::detail::pi::PiQueue PiQueue) {
if (has_property<ext::oneapi::property::queue::discard_events>() &&
has_property<property::queue::enable_profiling>()) {
throw sycl::exception(make_error_code(errc::invalid),
Expand All @@ -255,6 +224,42 @@ class queue_impl {
make_error_code(errc::invalid),
"Device provided by native Queue not found in Context.");
}
// The following commented section provides a guideline on how to use the
// TLS enabled mechanism to create a tracepoint and notify using XPTI. This
// is the prolog section and the epilog section will initiate the
// notification.
#if XPTI_ENABLE_INSTRUMENTATION
constexpr uint16_t NotificationTraceType =
static_cast<uint16_t>(xpti::trace_point_type_t::queue_create);
XPTIScope PrepareNotify((void *)this, NotificationTraceType,
SYCL_STREAM_NAME, "queue_create");
if (xptiCheckTraceEnabled(PrepareNotify.streamID(),
NotificationTraceType)) {
// Cache the trace event, stream id and instance IDs for the destructor
MTraceEvent = (void *)PrepareNotify.traceEvent();
MStreamID = PrepareNotify.streamID();
MInstanceID = PrepareNotify.instanceID();

// Add the function to capture meta data for the XPTI trace event
PrepareNotify.addMetadata([&](auto TEvent) {
xpti::addMetadata(TEvent, "sycl_context",
reinterpret_cast<size_t>(MContext->getHandleRef()));
if (MDevice) {
xpti::addMetadata(TEvent, "sycl_device_name",
MDevice->getDeviceName());
xpti::addMetadata(
TEvent, "sycl_device",
reinterpret_cast<size_t>(
MDevice->is_host() ? 0 : MDevice->getHandleRef()));
}
xpti::addMetadata(TEvent, "is_inorder", MIsInorder);
xpti::addMetadata(TEvent, "queue_id", MQueueID);
if (!MHostQueue)
xpti::addMetadata(TEvent, "queue_handle", getHandleRef());
});
PrepareNotify.notify();
}
#endif
}

public:
Expand All @@ -273,7 +278,9 @@ class queue_impl {
has_property<ext::oneapi::property::queue::discard_events>()),
MIsProfilingEnabled(has_property<property::queue::enable_profiling>()),
MHasDiscardEventsSupport(MDiscardEvents &&
(MHostQueue ? true : MIsInorder)) {
(MHostQueue ? true : MIsInorder)),
MQueueID{
MNextAvailableQueueID.fetch_add(1, std::memory_order_relaxed)} {
queue_impl_interop(PiQueue);
}

Expand Down Expand Up @@ -310,6 +317,7 @@ class queue_impl {
(xpti::trace_event_data_t *)MTraceEvent,
MInstanceID,
static_cast<const void *>("queue_destroy"));
xptiReleaseEvent((xpti::trace_event_data_t *)MTraceEvent);
}
#endif
throw_asynchronous();
Expand Down Expand Up @@ -695,6 +703,8 @@ class queue_impl {
return MGraph.lock();
}

unsigned long long getQueueID() { return MQueueID; }

protected:
// Hook to the scheduler to clean up any fusion command held on destruction.
void cleanup_fusion_cmd();
Expand Down Expand Up @@ -890,6 +900,9 @@ class queue_impl {
// recording commands to it.
std::weak_ptr<ext::oneapi::experimental::detail::graph_impl> MGraph{};

unsigned long long MQueueID;
static std::atomic<unsigned long long> MNextAvailableQueueID;

friend class sycl::ext::oneapi::experimental::detail::node_impl;
};

Expand Down
Loading

0 comments on commit da4b83c

Please sign in to comment.