Skip to content

Commit

Permalink
Userspace symbol resolving via addr2line
Browse files Browse the repository at this point in the history
This change implements additional symbol resolving for stack traces, which
improve the support for userspace stack traces.

A challenge with the stack traces obtained via ebpf is that they just contain
a raw address inside the virtual memory space of the process, which
requires some extra steps to be translated into a function name. This change
provides most of the infrastructure to provide the translation via a `DefaultResolver`
type which can resolve both kernelspace and userspace functions based on
a PID and address. In addition in adds a `SymbolResolver` trait that allows to customize
resolving behavior and extend it further.

I originally looked into just using the [StackTrace::resolve](https://docs.rs/aya/0.11.0/aya/maps/stack_trace/struct.StackTrace.html#method.resolve)
function. However I noticed that the information that is at the moment stored
in the StackTrace struct is not sufficient for resolving, e.g. due to missing the PID
and being unable to account for the virtual -> physical memory address translation.
Therefore this change uses a new resolver infrastructure.

Usage via a suitable EBPF program that sends stack traces and process IDs:

```rust

/// The BPF program populates this Queue with the process ID, kernel-space and user-space
/// stack trace IDs.
/// The former can be obtained via `bpf_get_current_pid_tgid()`, the stack
/// traces via `aya_bpf::maps::StackTrace`.
let mut stacks = Queue::<_, [u64; 3]>::try_from(bpf.map_mut("STACKS")?)?;
let stack_traces = StackTraceMap::try_from(bpf.map_mut("STACK_TRACES")?)?;
let resolver = DefaultResolver::new().unwrap();

loop {
    match stacks.pop(0) {
        Ok([pid_tgid, ktrace_id, utrace_id]) => {
            let tgid = pid_tgid & 0xFFFFFFFF;

            if let Ok(trace) = stack_traces.get(&(ktrace_id as u32), 0) {
                for f in trace.frames() {
                    let mut symbol = SymbolInfo::unresolved_kernel(f.ip);
                    resolver.resolve(&mut symbol);
                    println!("Resolved kernel address: 0x{:x} to {:?}", f.ip, symbol);
                }
            }

            if let Ok(trace) = stack_traces.get(&(utrace_id as u32), 0) {
                for f in trace.frames() {
                    let mut symbol = SymbolInfo::unresolved_user(tgid as _, f.ip);
                    resolver.resolve(&mut symbol);
                    info!("Resolved pid {}, address: 0x{:x} to {:?}", tgid, f.ip, symbol);
                }
            }
        }
    }
}
```

BPF code:
```rust
static STACK_TRACES: StackTrace = StackTrace::with_max_entries(10, 0);

static STACKS: Queue<[u64; 3]> = Queue::with_max_entries(1024, 0);

pub fn testprobe(ctx: ProbeContext) -> u32 {
    unsafe {
        let pid_tgid = bpf_get_current_pid_tgid();
        let ustack = STACK_TRACES.get_stackid(&ctx, BPF_F_USER_STACK as _);
        let kstack = STACK_TRACES.get_stackid(&ctx, 0);

        match (kstack, ustack) {
            (Ok(kstack), Ok(ustack)) => {
                if let Err(e) = STACKS.push(&[pid_tgid, kstack as _, ustack as _], 0) {
                    info!(&ctx, "Error pushing stack: {}", e);
                }
            },
            _ => {}
        }

        0
    }
}
```

Output when fetching stack traces of a kprobe on `sendmmsg` on a test program:
```
Resolved kernel address: 0xffffffff81a62691 to SymbolInfo { virtual_address: 18446744071589734033, object_address: Some(18446744071589734033), process_id: None, function_name: Some("__sys_sendmmsg"), object_path: None }
Resolved kernel address: 0xffffffff81a62870 to SymbolInfo { virtual_address: 18446744071589734512, object_address: Some(18446744071589734512), process_id: None, function_name: Some("__x64_sys_sendmmsg"), object_path: None }
Resolved kernel address: 0xffffffff81da30a3 to SymbolInfo { virtual_address: 18446744071593144483, object_address: Some(18446744071593144483), process_id: None, function_name: Some("do_syscall_64"), object_path: None }
Resolved kernel address: 0xffffffff81e0007c to SymbolInfo { virtual_address: 18446744071593525372, object_address: Some(18446744071593525372), process_id: None, function_name: Some("entry_SYSCALL_64_after_hwframe"), object_path: None }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x7fccf3b2adee to SymbolInfo { virtual_address: 140518238629358, object_address: Some(1195502), process_id: Some(22158), function_name: None, object_path: Some("/usr/lib/x86_64-linux-gnu/libc-2.31.so") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570cd66cc to SymbolInfo { virtual_address: 94856245241548, object_address: Some(947916), process_id: Some(22158), function_name: Some("tokio::io::async_fd::AsyncFdReadyGuard<Inner>::try_io"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570cd43e6 to SymbolInfo { virtual_address: 94856245232614, object_address: Some(938982), process_id: Some(22158), function_name: Some("quinn_udp::imp::UdpSocket::poll_send"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570ccd31f to SymbolInfo { virtual_address: 94856245203743, object_address: Some(910111), process_id: Some(22158), function_name: Some("<quinn::endpoint::EndpointDriver as core::future::future::Future>::poll"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570cc2400 to SymbolInfo { virtual_address: 94856245158912, object_address: Some(865280), process_id: Some(22158), function_name: Some("<core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570cd0e8d to SymbolInfo { virtual_address: 94856245218957, object_address: Some(925325), process_id: Some(22158), function_name: Some("tokio::runtime::task::harness::poll_future"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570cd160a to SymbolInfo { virtual_address: 94856245220874, object_address: Some(927242), process_id: Some(22158), function_name: Some("tokio::runtime::task::harness::Harness<T,S>::poll"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570c6ef25 to SymbolInfo { virtual_address: 94856244817701, object_address: Some(524069), process_id: Some(22158), function_name: Some("std::thread::local::LocalKey<T>::with"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570c7e505 to SymbolInfo { virtual_address: 94856244880645, object_address: Some(587013), process_id: Some(22158), function_name: Some("tokio::runtime::basic_scheduler::Context::run_task"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570c7b2fa to SymbolInfo { virtual_address: 94856244867834, object_address: Some(574202), process_id: Some(22158), function_name: Some("tokio::macros::scoped_tls::ScopedKey<T>::set"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570c7de7c to SymbolInfo { virtual_address: 94856244878972, object_address: Some(585340), process_id: Some(22158), function_name: Some("tokio::runtime::basic_scheduler::BasicScheduler::block_on"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570c75a3d to SymbolInfo { virtual_address: 94856244845117, object_address: Some(551485), process_id: Some(22158), function_name: Some("tokio::runtime::Runtime::block_on"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570c5c597 to SymbolInfo { virtual_address: 94856244741527, object_address: Some(447895), process_id: Some(22158), function_name: Some("std::sys_common::backtrace::__rust_begin_short_backtrace"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570c59d44 to SymbolInfo { virtual_address: 94856244731204, object_address: Some(437572), process_id: Some(22158), function_name: Some("core::ops::function::FnOnce::call_once{{vtable.shim}}"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
02:46:16 [INFO] ayatest: [ayatest/src/main.rs:98] Resolved pid 22158, address: 0x564570ecc763 to SymbolInfo { virtual_address: 94856247297891, object_address: Some(3004259), process_id: Some(22158), function_name: Some("std::sys::unix::thread::Thread::new::thread_start"), object_path: Some("/mnt/c/Users/matth/Code/rust/quinn/target/release/bulk") }
```
  • Loading branch information
Matthias247 committed Jul 4, 2022
1 parent 82cd3e6 commit 596db35
Show file tree
Hide file tree
Showing 3 changed files with 698 additions and 1 deletion.
4 changes: 3 additions & 1 deletion aya/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ documentation = "https://docs.rs/aya"
edition = "2018"

[dependencies]
addr2line = { version = "0.17", features = ["std-object"] }
libc = { version = "0.2.105" }
lru-cache = "0.1.2"
thiserror = "1"
object = { version = "0.28", default-features = false, features = ["std", "read_core", "elf"] }
object = { version = "0.29", default-features = false, features = ["std", "read_core", "elf"] }
bitflags = "1.2.1"
bytes = "1"
lazy_static = "1"
Expand Down
1 change: 1 addition & 0 deletions aya/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ mod generated;
pub mod maps;
mod obj;
pub mod programs;
pub mod symbols;
mod sys;
pub mod util;

Expand Down
Loading

0 comments on commit 596db35

Please sign in to comment.