diff --git a/bpf/errors.h b/bpf/errors.h new file mode 100644 index 000000000..74a31852d --- /dev/null +++ b/bpf/errors.h @@ -0,0 +1,27 @@ +#ifndef __ERRORS_H_ +#define __ERRORS_H_ + +#ifndef TASK_COMM_LEN +#define TASK_COMM_LEN 16 +#endif + +#ifndef ERR_MSG_LEN +#define ERR_MSG_LEN 128 +#endif + +#ifndef MAX_STACK_DEPTH +#define MAX_STACK_DEPTH 32 +#endif + +typedef __u64 stack_trace_t[MAX_STACK_DEPTH]; + +typedef struct error_event { + __u32 pid; + __u32 cpu_id; + char comm[TASK_COMM_LEN]; + __s32 ustack_sz; + stack_trace_t ustack; + u8 err_msg[ERR_MSG_LEN]; +} error_event; + +#endif /* __ERRORS_H_ */ diff --git a/bpf/go_nethttp.c b/bpf/go_nethttp.c index 90bf1a1fe..28e6f2b53 100644 --- a/bpf/go_nethttp.c +++ b/bpf/go_nethttp.c @@ -25,6 +25,7 @@ #include "tracing.h" #include "hpack.h" #include "ringbuf.h" +#include "errors.h" typedef struct http_func_invocation { u64 start_monotime_ns; @@ -46,6 +47,13 @@ struct { __uint(max_entries, MAX_CONCURRENT_REQUESTS); } ongoing_http_client_requests SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, void *); // key: pointer to the request goroutine + __type(value, struct error_event); + __uint(max_entries, MAX_CONCURRENT_REQUESTS); +} last_error SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); __type(key, void *); // key: pointer to the request goroutine @@ -198,6 +206,9 @@ int uprobe_ServeHTTPReturns(struct pt_regs *ctx) { resp_ptr = deref_resp_ptr; } + struct error_event *error = bpf_map_lookup_elem(&last_error, &goroutine_addr); + bpf_map_delete_elem(&last_error, &goroutine_addr); + http_request_trace *trace = bpf_ringbuf_reserve(&events, sizeof(http_request_trace), 0); if (!trace) { bpf_dbg_printk("can't reserve space in the ringbuffer"); @@ -208,6 +219,8 @@ int uprobe_ServeHTTPReturns(struct pt_regs *ctx) { trace->type = EVENT_HTTP_REQUEST; trace->start_monotime_ns = invocation->start_monotime_ns; trace->end_monotime_ns = bpf_ktime_get_ns(); + if (error) + trace->error = *error; goroutine_metadata *g_metadata = bpf_map_lookup_elem(&ongoing_goroutines, &goroutine_addr); if (g_metadata) { @@ -436,6 +449,75 @@ int uprobe_roundTripReturn(struct pt_regs *ctx) { return 0; } + +SEC("uprobe/error") +int uprobe_error(struct pt_regs *ctx) { + bpf_dbg_printk("=== uprobe/proc error === "); + + void *goroutine_addr = GOROUTINE_PTR(ctx); + bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); + + int pid = bpf_get_current_pid_tgid() >> 32; + int cpu_id = bpf_get_smp_processor_id(); + int BPF_F_USER_STACK = (1ULL << 8); + struct error_event event = { + .pid = pid, + .cpu_id = cpu_id, + }; + + if (bpf_get_current_comm(event.comm, sizeof(event.comm))) + event.comm[0] = 0; + + // Read the stack trace + event.ustack_sz = bpf_get_stack(ctx, event.ustack, sizeof(event.ustack), BPF_F_USER_STACK); + + // Get the caller of the error function and store it in the first slot of the stack + void *sp_caller = STACK_PTR(ctx); + u64 caller = 0; + bpf_probe_read(&caller, sizeof(u64), sp_caller); + bpf_dbg_printk("sp_caller %lx caller %lx", sp_caller, caller); + event.ustack[0] = caller; + + // Write event + if (bpf_map_update_elem(&last_error, &goroutine_addr, &event, BPF_ANY)) { + bpf_dbg_printk("can't update event error map element"); + } + return 0; +} + +SEC("uprobe/error_return") +int uprobe_errorReturn(struct pt_regs *ctx) { + bpf_dbg_printk("=== uprobe/proc error return === "); + + void *goroutine_addr = GOROUTINE_PTR(ctx); + bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); + + error_event *event = bpf_map_lookup_elem(&last_error, &goroutine_addr); + if (event == NULL) { + bpf_dbg_printk("can't read error event"); + return 0; + } + + // Read the error message + // GO_PARAM1(ctx) is the pointer to the error message + // GO_PARAM2(ctx) is the length of the error message + void *msg_ptr = GO_PARAM1(ctx); + u64 len = (u64)GO_PARAM2(ctx); + u64 max_size = sizeof(event->err_msg); + u64 size = max_size < len ? max_size : len; + bpf_probe_read(&event->err_msg, size, msg_ptr); + if (size < max_size) { + ((char *)event->err_msg)[size] = 0; + } + bpf_dbg_printk("error msg %llx, %s", msg_ptr, event->err_msg); + + // Write event + if (bpf_map_update_elem(&last_error, &goroutine_addr, event, BPF_ANY)) { + bpf_dbg_printk("can't update event error map element"); + } + return 0; +} + #ifndef NO_HEADER_PROPAGATION // Context propagation through HTTP headers SEC("uprobe/header_writeSubset") @@ -938,4 +1020,4 @@ int uprobe_queryReturn(struct pt_regs *ctx) { bpf_dbg_printk("can't reserve space in the ringbuffer"); } return 0; -} \ No newline at end of file +} diff --git a/bpf/go_nethttp.h b/bpf/go_nethttp.h index 18433b14f..4da6c9f1e 100644 --- a/bpf/go_nethttp.h +++ b/bpf/go_nethttp.h @@ -36,5 +36,4 @@ volatile const u64 rwc_conn_pos; volatile const u64 rws_conn_pos; volatile const u64 http2_server_conn_pos; volatile const u64 cc_tconn_pos; - #endif diff --git a/bpf/go_str.h b/bpf/go_str.h index 9428a45f0..c76d8c3fe 100644 --- a/bpf/go_str.h +++ b/bpf/go_str.h @@ -14,6 +14,7 @@ #define GO_STR_H #include "utils.h" +#include "bpf_dbg.h" static __always_inline int read_go_str_n(char *name, void *base_ptr, u64 len, void *field, u64 max_size) { u64 size = max_size < len ? max_size : len; diff --git a/bpf/headers/utils.h b/bpf/headers/utils.h index 37413daef..217ef671d 100644 --- a/bpf/headers/utils.h +++ b/bpf/headers/utils.h @@ -34,7 +34,7 @@ // In x86, current goroutine is pointed by r14, according to // https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-abi.md#amd64-architecture #define GOROUTINE_PTR(x) ((void*)(x)->r14) - +#define STACK_PTR(x) ((void*)(x)->sp) #elif defined(__TARGET_ARCH_arm64) #define GO_PARAM1(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[0]) @@ -50,6 +50,7 @@ // In arm64, current goroutine is pointed by R28 according to // https://github.com/golang/go/blob/master/src/cmd/compile/abi-internal.md#arm64-architecture #define GOROUTINE_PTR(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[28]) +#define STACK_PTR(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[13]) #endif /*defined(__TARGET_ARCH_arm64)*/ diff --git a/bpf/http_trace.h b/bpf/http_trace.h index 8efe99318..a8e25af98 100644 --- a/bpf/http_trace.h +++ b/bpf/http_trace.h @@ -15,6 +15,7 @@ #include "pid_types.h" #include "utils.h" +#include "errors.h" #include "http_types.h" #define PATH_MAX_LEN 100 @@ -39,6 +40,7 @@ typedef struct http_request_trace_t { u16 status; connection_info_t conn __attribute__ ((aligned (8))); s64 content_length; + error_event error; tp_info_t tp; pid_info pid; diff --git a/pkg/beyla/config.go b/pkg/beyla/config.go index a64001326..e78519250 100644 --- a/pkg/beyla/config.go +++ b/pkg/beyla/config.go @@ -73,11 +73,12 @@ var DefaultConfig = Config{ TTL: defaultMetricsTTL, }, Traces: otel.TracesConfig{ - Protocol: otel.ProtocolUnset, - TracesProtocol: otel.ProtocolUnset, - MaxQueueSize: 4096, - MaxExportBatchSize: 4096, - ReportersCacheLen: ReporterLRUSize, + Protocol: otel.ProtocolUnset, + TracesProtocol: otel.ProtocolUnset, + MaxQueueSize: 4096, + MaxExportBatchSize: 4096, + ReportersCacheLen: ReporterLRUSize, + ReportExceptionEvents: false, Instrumentations: []string{ instrumentations.InstrumentationALL, }, diff --git a/pkg/internal/discover/typer.go b/pkg/internal/discover/typer.go index 91b9c2ccc..df24e8fdf 100644 --- a/pkg/internal/discover/typer.go +++ b/pkg/internal/discover/typer.go @@ -166,7 +166,7 @@ func (t *typer) inspectOffsets(execElf *exec.FileInfo) (*goexec.Offsets, bool, e t.log.Debug("skipping inspection for Go functions", "pid", execElf.Pid, "comm", execElf.CmdExePath) } else { t.log.Debug("inspecting", "pid", execElf.Pid, "comm", execElf.CmdExePath) - offsets, err := goexec.InspectOffsets(execElf, t.allGoFunctions) + offsets, err := goexec.InspectOffsets(&t.cfg.Traces, execElf, t.allGoFunctions) if err != nil { t.log.Debug("couldn't find go specific tracers", "error", err) return nil, false, err diff --git a/pkg/internal/ebpf/common/bpf_bpf.o b/pkg/internal/ebpf/common/bpf_bpf.o new file mode 100644 index 000000000..d175481fa Binary files /dev/null and b/pkg/internal/ebpf/common/bpf_bpf.o differ diff --git a/pkg/internal/ebpf/common/bpf_bpfel_arm64.go b/pkg/internal/ebpf/common/bpf_bpfel_arm64.go index 85de02229..f346eb893 100644 --- a/pkg/internal/ebpf/common/bpf_bpfel_arm64.go +++ b/pkg/internal/ebpf/common/bpf_bpfel_arm64.go @@ -90,7 +90,16 @@ type bpfHttpRequestTrace struct { _ [2]byte Conn bpfConnectionInfoT ContentLength int64 - Tp struct { + Error struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 + } + Tp struct { TraceId [16]uint8 SpanId [8]uint8 ParentId [8]uint8 diff --git a/pkg/internal/ebpf/common/bpf_bpfel_arm64.o b/pkg/internal/ebpf/common/bpf_bpfel_arm64.o index 5d3393e13..9b1be57a0 100644 Binary files a/pkg/internal/ebpf/common/bpf_bpfel_arm64.o and b/pkg/internal/ebpf/common/bpf_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/common/bpf_bpfel_x86.go b/pkg/internal/ebpf/common/bpf_bpfel_x86.go index 79fb03529..e6cd8107d 100644 --- a/pkg/internal/ebpf/common/bpf_bpfel_x86.go +++ b/pkg/internal/ebpf/common/bpf_bpfel_x86.go @@ -90,7 +90,16 @@ type bpfHttpRequestTrace struct { _ [2]byte Conn bpfConnectionInfoT ContentLength int64 - Tp struct { + Error struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 + } + Tp struct { TraceId [16]uint8 SpanId [8]uint8 ParentId [8]uint8 diff --git a/pkg/internal/ebpf/common/bpf_bpfel_x86.o b/pkg/internal/ebpf/common/bpf_bpfel_x86.o index 5d3393e13..9b1be57a0 100644 Binary files a/pkg/internal/ebpf/common/bpf_bpfel_x86.o and b/pkg/internal/ebpf/common/bpf_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/common/common.go b/pkg/internal/ebpf/common/common.go index cf2ccf370..be008592c 100644 --- a/pkg/internal/ebpf/common/common.go +++ b/pkg/internal/ebpf/common/common.go @@ -136,8 +136,7 @@ func ReadBPFTraceAsSpan(record *ringbuf.Record, filter ServiceFilter) (request.S if err != nil { return request.Span{}, true, err } - - return HTTPRequestTraceToSpan(&event), false, nil + return HTTPRequestTraceToSpan(&event, filter), false, nil } func ReadSQLRequestTraceAsSpan(record *ringbuf.Record) (request.Span, bool, error) { diff --git a/pkg/internal/ebpf/common/pids.go b/pkg/internal/ebpf/common/pids.go index fbc76dae5..1c0b6f6db 100644 --- a/pkg/internal/ebpf/common/pids.go +++ b/pkg/internal/ebpf/common/pids.go @@ -1,6 +1,7 @@ package ebpfcommon import ( + "debug/gosym" "log/slog" "sync" @@ -31,11 +32,12 @@ type PIDInfo struct { } type ServiceFilter interface { - AllowPID(uint32, uint32, svc.ID, PIDType) + AllowPID(uint32, uint32, svc.ID, PIDType, *gosym.Table) BlockPID(uint32, uint32) ValidPID(uint32, uint32, PIDType) bool Filter(inputSpans []request.Span) []request.Span CurrentPIDs(PIDType) map[uint32]map[uint32]svc.ID + GetSymTab(uint32) *gosym.Table } // PIDsFilter keeps a thread-safe copy of the PIDs whose traces are allowed to @@ -44,6 +46,7 @@ type ServiceFilter interface { type PIDsFilter struct { log *slog.Logger current map[uint32]map[uint32]PIDInfo + symTabs map[uint32]*gosym.Table mux *sync.RWMutex } @@ -55,6 +58,7 @@ func NewPIDsFilter(log *slog.Logger) *PIDsFilter { log: log, current: map[uint32]map[uint32]PIDInfo{}, mux: &sync.RWMutex{}, + symTabs: make(map[uint32]*gosym.Table), } } @@ -73,16 +77,17 @@ func CommonPIDsFilter(systemWide bool) ServiceFilter { return commonPIDsFilter } -func (pf *PIDsFilter) AllowPID(pid, ns uint32, svc svc.ID, pidType PIDType) { +func (pf *PIDsFilter) AllowPID(pid, ns uint32, svc svc.ID, pidType PIDType, symTab *gosym.Table) { pf.mux.Lock() defer pf.mux.Unlock() - pf.addPID(pid, ns, svc, pidType) + pf.addPID(pid, ns, svc, pidType, symTab) } func (pf *PIDsFilter) BlockPID(pid, ns uint32) { pf.mux.Lock() defer pf.mux.Unlock() pf.removePID(pid, ns) + } func (pf *PIDsFilter) ValidPID(userPID, ns uint32, pidType PIDType) bool { @@ -151,13 +156,24 @@ func (pf *PIDsFilter) Filter(inputSpans []request.Span) []request.Span { return outputSpans } -func (pf *PIDsFilter) addPID(pid, nsid uint32, s svc.ID, t PIDType) { +func (pf *PIDsFilter) GetSymTab(pid uint32) *gosym.Table { + pf.mux.RLock() + defer pf.mux.RUnlock() + return pf.symTabs[pid] +} + +func (pf *PIDsFilter) addPID(pid, nsid uint32, s svc.ID, t PIDType, symTab *gosym.Table) { ns, nsExists := pf.current[nsid] if !nsExists { ns = make(map[uint32]PIDInfo) pf.current[nsid] = ns } + _, stExists := pf.symTabs[pid] + if !stExists { + pf.symTabs[pid] = symTab + } + allPids, err := readNamespacePIDs(int32(pid)) if err != nil { @@ -180,13 +196,14 @@ func (pf *PIDsFilter) removePID(pid, nsid uint32) { if len(ns) == 0 { delete(pf.current, nsid) } + delete(pf.symTabs, pid) } // IdentityPidsFilter is a PIDsFilter that does not filter anything. It is feasible // for system-wide instrumenation type IdentityPidsFilter struct{} -func (pf *IdentityPidsFilter) AllowPID(_ uint32, _ uint32, _ svc.ID, _ PIDType) {} +func (pf *IdentityPidsFilter) AllowPID(_ uint32, _ uint32, _ svc.ID, _ PIDType, _ *gosym.Table) {} func (pf *IdentityPidsFilter) BlockPID(_ uint32, _ uint32) {} @@ -206,6 +223,10 @@ func (pf *IdentityPidsFilter) Filter(inputSpans []request.Span) []request.Span { return inputSpans } +func (pf *IdentityPidsFilter) GetSymTab(_ uint32) *gosym.Table { + return nil +} + func serviceInfo(pid uint32) svc.ID { cached, ok := activePids.Get(pid) if ok { diff --git a/pkg/internal/ebpf/common/pids_test.go b/pkg/internal/ebpf/common/pids_test.go index 350338f10..5736de7e0 100644 --- a/pkg/internal/ebpf/common/pids_test.go +++ b/pkg/internal/ebpf/common/pids_test.go @@ -25,9 +25,9 @@ func TestFilter_SameNS(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := NewPIDsFilter(slog.With("env", "testing")) - pf.AllowPID(123, 33, svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 33, svc.ID{}, PIDTypeGo) - pf.AllowPID(789, 33, svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 33, svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 33, svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(789, 33, svc.ID{}, PIDTypeGo, nil) // with the same namespace, it filters by user PID, as it is the PID // that is seen by Beyla's process discovery @@ -43,9 +43,9 @@ func TestFilter_DifferentNS(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := NewPIDsFilter(slog.With("env", "testing")) - pf.AllowPID(123, 22, svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 22, svc.ID{}, PIDTypeGo) - pf.AllowPID(666, 22, svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 22, svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 22, svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(666, 22, svc.ID{}, PIDTypeGo, nil) // with the same namespace, it filters by user PID, as it is the PID // that is seen by Beyla's process discovery @@ -57,8 +57,8 @@ func TestFilter_Block(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := NewPIDsFilter(slog.With("env", "testing")) - pf.AllowPID(123, 33, svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 33, svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 33, svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 33, svc.ID{}, PIDTypeGo, nil) pf.BlockPID(123, 33) // with the same namespace, it filters by user PID, as it is the PID @@ -75,9 +75,9 @@ func TestFilter_NewNSLater(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := NewPIDsFilter(slog.With("env", "testing")) - pf.AllowPID(123, 33, svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 33, svc.ID{}, PIDTypeGo) - pf.AllowPID(789, 33, svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 33, svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 33, svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(789, 33, svc.ID{}, PIDTypeGo, nil) // with the same namespace, it filters by user PID, as it is the PID // that is seen by Beyla's process discovery @@ -87,7 +87,7 @@ func TestFilter_NewNSLater(t *testing.T) { {Pid: request.PidInfo{UserPID: 789, HostPID: 234, Namespace: 33}}, }, pf.Filter(spanSet)) - pf.AllowPID(1000, 44, svc.ID{}, PIDTypeGo) + pf.AllowPID(1000, 44, svc.ID{}, PIDTypeGo, nil) assert.Equal(t, []request.Span{ {Pid: request.PidInfo{UserPID: 123, HostPID: 333, Namespace: 33}}, diff --git a/pkg/internal/ebpf/common/ringbuf_test.go b/pkg/internal/ebpf/common/ringbuf_test.go index eb82e934d..ab4db8a52 100644 --- a/pkg/internal/ebpf/common/ringbuf_test.go +++ b/pkg/internal/ebpf/common/ringbuf_test.go @@ -3,6 +3,7 @@ package ebpfcommon import ( "bytes" "context" + "debug/gosym" "encoding/binary" "log/slog" "sync" @@ -31,7 +32,7 @@ func TestForwardRingbuf_CapacityFull(t *testing.T) { metrics := &metricsReporter{} forwardedMessages := make(chan []request.Span, 100) fltr := TestPidsFilter{services: map[uint32]svc.ID{}} - fltr.AllowPID(1, 1, svc.ID{Name: "myService"}, PIDTypeGo) + fltr.AllowPID(1, 1, svc.ID{Name: "myService"}, PIDTypeGo, nil) go ForwardRingbuf( &TracerConfig{BatchLength: 10}, nil, // the source ring buffer can be null @@ -83,7 +84,7 @@ func TestForwardRingbuf_Deadline(t *testing.T) { metrics := &metricsReporter{} forwardedMessages := make(chan []request.Span, 100) fltr := TestPidsFilter{services: map[uint32]svc.ID{}} - fltr.AllowPID(1, 1, svc.ID{Name: "myService"}, PIDTypeGo) + fltr.AllowPID(1, 1, svc.ID{Name: "myService"}, PIDTypeGo, nil) go ForwardRingbuf( &TracerConfig{BatchLength: 10, BatchTimeout: 20 * time.Millisecond}, nil, // the source ring buffer can be null @@ -219,7 +220,7 @@ type TestPidsFilter struct { services map[uint32]svc.ID } -func (pf *TestPidsFilter) AllowPID(p uint32, _ uint32, s svc.ID, _ PIDType) { +func (pf *TestPidsFilter) AllowPID(p uint32, _ uint32, s svc.ID, _ PIDType, _ *gosym.Table) { pf.services[p] = s } @@ -235,6 +236,10 @@ func (pf *TestPidsFilter) CurrentPIDs(_ PIDType) map[uint32]map[uint32]svc.ID { return nil } +func (pf *TestPidsFilter) GetSymTab(_ uint32) *gosym.Table { + return nil +} + func (pf *TestPidsFilter) Filter(inputSpans []request.Span) []request.Span { for i := range inputSpans { s := &inputSpans[i] diff --git a/pkg/internal/ebpf/common/spanner.go b/pkg/internal/ebpf/common/spanner.go index b4bcb1d87..0866ee254 100644 --- a/pkg/internal/ebpf/common/spanner.go +++ b/pkg/internal/ebpf/common/spanner.go @@ -2,7 +2,10 @@ package ebpfcommon import ( "bytes" + "debug/gosym" + "fmt" "log/slog" + "strings" "unsafe" trace2 "go.opentelemetry.io/otel/trace" @@ -13,7 +16,7 @@ import ( var log = slog.With("component", "goexec.spanner") -func HTTPRequestTraceToSpan(trace *HTTPRequestTrace) request.Span { +func HTTPRequestTraceToSpan(trace *HTTPRequestTrace, filter ServiceFilter) request.Span { // From C, assuming 0-ended strings methodLen := bytes.IndexByte(trace.Method[:], 0) if methodLen < 0 { @@ -24,6 +27,10 @@ func HTTPRequestTraceToSpan(trace *HTTPRequestTrace) request.Span { if pathLen < 0 { pathLen = len(trace.Path) } + errMsgLen := bytes.IndexByte(trace.Error.ErrMsg[:], 0) + if errMsgLen < 0 { + errMsgLen = len(trace.Error.ErrMsg) + } path := string(trace.Path[:pathLen]) peer := "" @@ -58,7 +65,25 @@ func HTTPRequestTraceToSpan(trace *HTTPRequestTrace) request.Span { UserPID: trace.Pid.UserPid, Namespace: trace.Pid.Ns, }, + ErrorMessage: string(trace.Error.ErrMsg[:errMsgLen]), + ErrorStacktrace: extractErrorStacktrace(trace, filter.GetSymTab(trace.Pid.UserPid)), + } +} + +func extractErrorStacktrace(trace *HTTPRequestTrace, symTab *gosym.Table) string { + var stacktrace strings.Builder + if symTab != nil && trace.Error.UstackSz > 0 { + for _, pc := range trace.Error.Ustack { + f := symTab.PCToFunc(pc) + if f == nil { + break + } + file, line, _ := symTab.PCToLine(pc) + stacktrace.WriteString(fmt.Sprintf("%s\n", f.Name)) + stacktrace.WriteString(fmt.Sprintf("\t%s:%d\n", file, line)) + } } + return stacktrace.String() } func SQLRequestTraceToSpan(trace *SQLRequestTrace) request.Span { diff --git a/pkg/internal/ebpf/common/spanner_test.go b/pkg/internal/ebpf/common/spanner_test.go index 00ff9538e..4062f5e3b 100644 --- a/pkg/internal/ebpf/common/spanner_test.go +++ b/pkg/internal/ebpf/common/spanner_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/grafana/beyla/pkg/internal/request" + "github.com/grafana/beyla/pkg/internal/svc" ) func tocstr(s string) []byte { @@ -54,38 +55,40 @@ func assertMatches(t *testing.T, span *request.Span, method, path string, status } func TestRequestTraceParsing(t *testing.T) { + fltr := TestPidsFilter{services: map[uint32]svc.ID{}} t.Run("Test basic parsing", func(t *testing.T) { tr := makeHTTPRequestTrace("POST", "/users", 200, 5) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "POST", "/users", 200, 5) }) t.Run("Test with empty path and missing peer host", func(t *testing.T) { tr := makeHTTPRequestTrace("GET", "", 403, 6) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "GET", "", 403, 6) }) t.Run("Test with missing peer port", func(t *testing.T) { tr := makeHTTPRequestTrace("GET", "/posts/1/1", 500, 1) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "GET", "/posts/1/1", 500, 1) }) t.Run("Test with invalid peer port", func(t *testing.T) { tr := makeHTTPRequestTrace("GET", "/posts/1/1", 500, 1) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "GET", "/posts/1/1", 500, 1) }) t.Run("Test with GRPC request", func(t *testing.T) { tr := makeGRPCRequestTrace("/posts/1/1", 2, 1) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "", "/posts/1/1", 2, 1) }) } func makeSpanWithTimings(goStart, start, end uint64) request.Span { + fltr := TestPidsFilter{services: map[uint32]svc.ID{}} tr := HTTPRequestTrace{ Type: 1, Path: [100]uint8{}, @@ -95,7 +98,7 @@ func makeSpanWithTimings(goStart, start, end uint64) request.Span { EndMonotimeNs: end, } - return HTTPRequestTraceToSpan(&tr) + return HTTPRequestTraceToSpan(&tr, &fltr) } func TestSpanNesting(t *testing.T) { diff --git a/pkg/internal/ebpf/goredis/goredis.go b/pkg/internal/ebpf/goredis/goredis.go index a7f17c0a3..8379b0bc3 100644 --- a/pkg/internal/ebpf/goredis/goredis.go +++ b/pkg/internal/ebpf/goredis/goredis.go @@ -14,6 +14,7 @@ package goredis import ( "context" + "debug/gosym" "io" "log/slog" "unsafe" @@ -51,8 +52,8 @@ func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo, nil) } func (p *Tracer) BlockPID(pid, ns uint32) { diff --git a/pkg/internal/ebpf/goruntime/goruntime.go b/pkg/internal/ebpf/goruntime/goruntime.go index cf2073945..79ba14979 100644 --- a/pkg/internal/ebpf/goruntime/goruntime.go +++ b/pkg/internal/ebpf/goruntime/goruntime.go @@ -14,6 +14,7 @@ package goruntime import ( "context" + "debug/gosym" "io" "log/slog" @@ -50,8 +51,8 @@ func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo, nil) } func (p *Tracer) BlockPID(pid, ns uint32) { diff --git a/pkg/internal/ebpf/grpc/bpf_bpfel_arm64.o b/pkg/internal/ebpf/grpc/bpf_bpfel_arm64.o index ba3250f8f..31761b921 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_bpfel_arm64.o and b/pkg/internal/ebpf/grpc/bpf_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/grpc/bpf_bpfel_x86.o b/pkg/internal/ebpf/grpc/bpf_bpfel_x86.o index cb1502622..a1a3189da 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_bpfel_x86.o and b/pkg/internal/ebpf/grpc/bpf_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/grpc/bpf_debug_bpfel_arm64.o b/pkg/internal/ebpf/grpc/bpf_debug_bpfel_arm64.o index 75f3b6314..f8a6944cc 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_debug_bpfel_arm64.o and b/pkg/internal/ebpf/grpc/bpf_debug_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/grpc/bpf_debug_bpfel_x86.o b/pkg/internal/ebpf/grpc/bpf_debug_bpfel_x86.o index 8de127fcf..42a5bb625 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_debug_bpfel_x86.o and b/pkg/internal/ebpf/grpc/bpf_debug_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/grpc/bpf_tp_bpfel_arm64.o b/pkg/internal/ebpf/grpc/bpf_tp_bpfel_arm64.o index cea9e2994..c4e05336e 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_tp_bpfel_arm64.o and b/pkg/internal/ebpf/grpc/bpf_tp_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/grpc/bpf_tp_bpfel_x86.o b/pkg/internal/ebpf/grpc/bpf_tp_bpfel_x86.o index a89cf0324..7d6985c94 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_tp_bpfel_x86.o and b/pkg/internal/ebpf/grpc/bpf_tp_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_arm64.o b/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_arm64.o index 6c8763e59..ec4973eef 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_arm64.o and b/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_x86.o b/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_x86.o index 0f4b745fb..5b7295ce1 100644 Binary files a/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_x86.o and b/pkg/internal/ebpf/grpc/bpf_tp_debug_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/grpc/grpc.go b/pkg/internal/ebpf/grpc/grpc.go index a67f07e33..88b8259b1 100644 --- a/pkg/internal/ebpf/grpc/grpc.go +++ b/pkg/internal/ebpf/grpc/grpc.go @@ -16,6 +16,7 @@ package grpc import ( "context" + "debug/gosym" "io" "log/slog" "unsafe" @@ -56,8 +57,8 @@ func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo, nil) } func (p *Tracer) BlockPID(pid, ns uint32) { diff --git a/pkg/internal/ebpf/httpfltr/httpfltr.go b/pkg/internal/ebpf/httpfltr/httpfltr.go index 39828196b..40958beaa 100644 --- a/pkg/internal/ebpf/httpfltr/httpfltr.go +++ b/pkg/internal/ebpf/httpfltr/httpfltr.go @@ -2,6 +2,7 @@ package httpfltr import ( "context" + "debug/gosym" "io" "log/slog" "time" @@ -95,8 +96,8 @@ func (p *Tracer) rebuildValidPids() { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes, nil) p.rebuildValidPids() } diff --git a/pkg/internal/ebpf/httpssl/httpssl.go b/pkg/internal/ebpf/httpssl/httpssl.go index e644a5b9c..c952a7462 100644 --- a/pkg/internal/ebpf/httpssl/httpssl.go +++ b/pkg/internal/ebpf/httpssl/httpssl.go @@ -2,6 +2,7 @@ package httpssl import ( "context" + "debug/gosym" "io" "log/slog" "sync" @@ -46,8 +47,8 @@ func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes, nil) } func (p *Tracer) BlockPID(pid, ns uint32) { diff --git a/pkg/internal/ebpf/kafkago/kafkago.go b/pkg/internal/ebpf/kafkago/kafkago.go index a484ffa99..c1fa84c20 100644 --- a/pkg/internal/ebpf/kafkago/kafkago.go +++ b/pkg/internal/ebpf/kafkago/kafkago.go @@ -14,6 +14,7 @@ package kafkago import ( "context" + "debug/gosym" "io" "log/slog" "unsafe" @@ -51,8 +52,8 @@ func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo, nil) } func (p *Tracer) BlockPID(pid, ns uint32) { diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.go index 93f9d6317..39952d162 100644 --- a/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.go +++ b/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.go @@ -19,6 +19,16 @@ type bpfConnectionInfoT struct { D_port uint16 } +type bpfErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpfGoroutineMetadata struct { Parent uint64 Timestamp uint64 @@ -122,6 +132,8 @@ type bpfProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -144,6 +156,7 @@ type bpfMapSpecs struct { Events *ebpf.MapSpec `ebpf:"events"` GoTraceMap *ebpf.MapSpec `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -176,6 +189,7 @@ type bpfMaps struct { Events *ebpf.Map `ebpf:"events"` GoTraceMap *ebpf.Map `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -191,6 +205,7 @@ func (m *bpfMaps) Close() error { m.Events, m.GoTraceMap, m.GolangMapbucketStorageMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -210,6 +225,8 @@ type bpfPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -231,6 +248,8 @@ func (p *bpfPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.o index 84830675e..1fac51409 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.o and b/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.go index f0bb71eaf..e790058b8 100644 --- a/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.go +++ b/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.go @@ -19,6 +19,16 @@ type bpfConnectionInfoT struct { D_port uint16 } +type bpfErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpfGoroutineMetadata struct { Parent uint64 Timestamp uint64 @@ -122,6 +132,8 @@ type bpfProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -144,6 +156,7 @@ type bpfMapSpecs struct { Events *ebpf.MapSpec `ebpf:"events"` GoTraceMap *ebpf.MapSpec `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -176,6 +189,7 @@ type bpfMaps struct { Events *ebpf.Map `ebpf:"events"` GoTraceMap *ebpf.Map `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -191,6 +205,7 @@ func (m *bpfMaps) Close() error { m.Events, m.GoTraceMap, m.GolangMapbucketStorageMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -210,6 +225,8 @@ type bpfPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -231,6 +248,8 @@ func (p *bpfPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.o index 1b6ddb89a..3e5ab2c97 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.o and b/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.go index 726ab6b0d..28300c451 100644 --- a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.go +++ b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.go @@ -19,6 +19,16 @@ type bpf_debugConnectionInfoT struct { D_port uint16 } +type bpf_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_debugGoroutineMetadata struct { Parent uint64 Timestamp uint64 @@ -122,6 +132,8 @@ type bpf_debugProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -145,6 +157,7 @@ type bpf_debugMapSpecs struct { Events *ebpf.MapSpec `ebpf:"events"` GoTraceMap *ebpf.MapSpec `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -178,6 +191,7 @@ type bpf_debugMaps struct { Events *ebpf.Map `ebpf:"events"` GoTraceMap *ebpf.Map `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -194,6 +208,7 @@ func (m *bpf_debugMaps) Close() error { m.Events, m.GoTraceMap, m.GolangMapbucketStorageMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -213,6 +228,8 @@ type bpf_debugPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -234,6 +251,8 @@ func (p *bpf_debugPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.o index cd7b0fedd..c06d3b7ce 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.o and b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.go index a9d759052..d37f09afb 100644 --- a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.go +++ b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.go @@ -19,6 +19,16 @@ type bpf_debugConnectionInfoT struct { D_port uint16 } +type bpf_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_debugGoroutineMetadata struct { Parent uint64 Timestamp uint64 @@ -122,6 +132,8 @@ type bpf_debugProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -145,6 +157,7 @@ type bpf_debugMapSpecs struct { Events *ebpf.MapSpec `ebpf:"events"` GoTraceMap *ebpf.MapSpec `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -178,6 +191,7 @@ type bpf_debugMaps struct { Events *ebpf.Map `ebpf:"events"` GoTraceMap *ebpf.Map `ebpf:"go_trace_map"` GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -194,6 +208,7 @@ func (m *bpf_debugMaps) Close() error { m.Events, m.GoTraceMap, m.GolangMapbucketStorageMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -213,6 +228,8 @@ type bpf_debugPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -234,6 +251,8 @@ func (p *bpf_debugPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.o index 16ec2602c..9b40cbaac 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.o and b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.go index a92c639b8..0e2130306 100644 --- a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.go +++ b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.go @@ -19,6 +19,16 @@ type bpf_tpConnectionInfoT struct { D_port uint16 } +type bpf_tpErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tpFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tpTpInfoT @@ -128,6 +138,8 @@ type bpf_tpProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -153,6 +165,7 @@ type bpf_tpMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.MapSpec `ebpf:"header_req_map"` Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -188,6 +201,7 @@ type bpf_tpMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.Map `ebpf:"header_req_map"` Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -206,6 +220,7 @@ func (m *bpf_tpMaps) Close() error { m.GolangMapbucketStorageMap, m.HeaderReqMap, m.Http2ReqMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -225,6 +240,8 @@ type bpf_tpPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -246,6 +263,8 @@ func (p *bpf_tpPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.o index 7845b8b23..cd772e86d 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.o and b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.go index a8d396949..afc4dedfe 100644 --- a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.go +++ b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.go @@ -19,6 +19,16 @@ type bpf_tpConnectionInfoT struct { D_port uint16 } +type bpf_tpErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tpFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tpTpInfoT @@ -128,6 +138,8 @@ type bpf_tpProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -153,6 +165,7 @@ type bpf_tpMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.MapSpec `ebpf:"header_req_map"` Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -188,6 +201,7 @@ type bpf_tpMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.Map `ebpf:"header_req_map"` Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -206,6 +220,7 @@ func (m *bpf_tpMaps) Close() error { m.GolangMapbucketStorageMap, m.HeaderReqMap, m.Http2ReqMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -225,6 +240,8 @@ type bpf_tpPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -246,6 +263,8 @@ func (p *bpf_tpPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.o index d50a242a5..7a826f046 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.o and b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.go index f3b070c05..915d20a77 100644 --- a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.go +++ b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.go @@ -19,6 +19,16 @@ type bpf_tp_debugConnectionInfoT struct { D_port uint16 } +type bpf_tp_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tp_debugFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tp_debugTpInfoT @@ -128,6 +138,8 @@ type bpf_tp_debugProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -154,6 +166,7 @@ type bpf_tp_debugMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.MapSpec `ebpf:"header_req_map"` Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -190,6 +203,7 @@ type bpf_tp_debugMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.Map `ebpf:"header_req_map"` Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -209,6 +223,7 @@ func (m *bpf_tp_debugMaps) Close() error { m.GolangMapbucketStorageMap, m.HeaderReqMap, m.Http2ReqMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -228,6 +243,8 @@ type bpf_tp_debugPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -249,6 +266,8 @@ func (p *bpf_tp_debugPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.o index 1c3fd0aa8..a7792295e 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.o and b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.o differ diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.go index 7e78c639c..28d8dea5e 100644 --- a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.go +++ b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.go @@ -19,6 +19,16 @@ type bpf_tp_debugConnectionInfoT struct { D_port uint16 } +type bpf_tp_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tp_debugFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tp_debugTpInfoT @@ -128,6 +138,8 @@ type bpf_tp_debugProgramSpecs struct { UprobeServeHTTPReturns *ebpf.ProgramSpec `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -154,6 +166,7 @@ type bpf_tp_debugMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.MapSpec `ebpf:"header_req_map"` Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` + LastError *ebpf.MapSpec `ebpf:"last_error"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.MapSpec `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.MapSpec `ebpf:"ongoing_http_client_requests"` @@ -190,6 +203,7 @@ type bpf_tp_debugMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` HeaderReqMap *ebpf.Map `ebpf:"header_req_map"` Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` + LastError *ebpf.Map `ebpf:"last_error"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoroutines *ebpf.Map `ebpf:"ongoing_goroutines"` OngoingHttpClientRequests *ebpf.Map `ebpf:"ongoing_http_client_requests"` @@ -209,6 +223,7 @@ func (m *bpf_tp_debugMaps) Close() error { m.GolangMapbucketStorageMap, m.HeaderReqMap, m.Http2ReqMap, + m.LastError, m.OngoingClientConnections, m.OngoingGoroutines, m.OngoingHttpClientRequests, @@ -228,6 +243,8 @@ type bpf_tp_debugPrograms struct { UprobeServeHTTPReturns *ebpf.Program `ebpf:"uprobe_ServeHTTPReturns"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeHttp2FramerWriteHeaders *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders"` UprobeHttp2FramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_http2FramerWriteHeaders_returns"` @@ -249,6 +266,8 @@ func (p *bpf_tp_debugPrograms) Close() error { p.UprobeServeHTTPReturns, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeHttp2FramerWriteHeaders, p.UprobeHttp2FramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.o index 57a7e806e..351180293 100644 Binary files a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.o and b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.o differ diff --git a/pkg/internal/ebpf/nethttp/nethttp.go b/pkg/internal/ebpf/nethttp/nethttp.go index ac7c624c5..2354b5235 100644 --- a/pkg/internal/ebpf/nethttp/nethttp.go +++ b/pkg/internal/ebpf/nethttp/nethttp.go @@ -16,6 +16,7 @@ package nethttp import ( "context" + "debug/gosym" "io" "log/slog" "unsafe" @@ -37,26 +38,28 @@ import ( //go:generate $BPF2GO -cc $BPF_CLANG -cflags $BPF_CFLAGS -target amd64,arm64 bpf_tp_debug ../../../../bpf/go_nethttp.c -- -I../../../../bpf/headers -DBPF_DEBUG type Tracer struct { - log *slog.Logger - pidsFilter ebpfcommon.ServiceFilter - cfg *ebpfcommon.TracerConfig - metrics imetrics.Reporter - bpfObjects bpfObjects - closers []io.Closer + log *slog.Logger + pidsFilter ebpfcommon.ServiceFilter + cfg *ebpfcommon.TracerConfig + reportErrors bool + metrics imetrics.Reporter + bpfObjects bpfObjects + closers []io.Closer } func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { log := slog.With("component", "nethttp.Tracer") return &Tracer{ - log: log, - pidsFilter: ebpfcommon.CommonPIDsFilter(cfg.Discovery.SystemWide), - cfg: &cfg.EBPF, - metrics: metrics, + log: log, + pidsFilter: ebpfcommon.CommonPIDsFilter(cfg.Discovery.SystemWide), + cfg: &cfg.EBPF, + reportErrors: cfg.Traces.ReportExceptionEvents, + metrics: metrics, } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, symTab *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo, symTab) } func (p *Tracer) BlockPID(pid, ns uint32) { @@ -196,6 +199,15 @@ func (p *Tracer) GoProbes() map[string]ebpfcommon.FunctionPrograms { } } + if p.reportErrors { + m["fmt.Errorf"] = ebpfcommon.FunctionPrograms{ + Start: p.bpfObjects.UprobeError, + } + m["errors.(*errorString).Error"] = ebpfcommon.FunctionPrograms{ + End: p.bpfObjects.UprobeErrorReturn, + } + } + return m } diff --git a/pkg/internal/ebpf/nodejs/nodejs.go b/pkg/internal/ebpf/nodejs/nodejs.go index dcaeed220..fdea008c3 100644 --- a/pkg/internal/ebpf/nodejs/nodejs.go +++ b/pkg/internal/ebpf/nodejs/nodejs.go @@ -2,6 +2,7 @@ package nodejs import ( "context" + "debug/gosym" "io" "log/slog" @@ -39,8 +40,8 @@ func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes, nil) } func (p *Tracer) BlockPID(pid, ns uint32) { diff --git a/pkg/internal/ebpf/sarama/sarama.go b/pkg/internal/ebpf/sarama/sarama.go index aefb23023..de5ed772b 100644 --- a/pkg/internal/ebpf/sarama/sarama.go +++ b/pkg/internal/ebpf/sarama/sarama.go @@ -14,6 +14,7 @@ package sarama import ( "context" + "debug/gosym" "io" "log/slog" "unsafe" @@ -51,8 +52,8 @@ func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo) +func (p *Tracer) AllowPID(pid, ns uint32, svc svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo, nil) } func (p *Tracer) BlockPID(pid, ns uint32) { diff --git a/pkg/internal/ebpf/tracer.go b/pkg/internal/ebpf/tracer.go index ffc51d094..2e5fe1a0b 100644 --- a/pkg/internal/ebpf/tracer.go +++ b/pkg/internal/ebpf/tracer.go @@ -2,6 +2,7 @@ package ebpf import ( "context" + "debug/gosym" "io" "log/slog" @@ -19,7 +20,7 @@ type PIDsAccounter interface { // AllowPID notifies the tracer to accept traces from the process with the // provided PID. Unless system-wide instrumentation, the Tracer should discard // traces from processes whose PID has not been allowed before - AllowPID(uint32, uint32, svc.ID) + AllowPID(uint32, uint32, svc.ID, *gosym.Table) // BlockPID notifies the tracer to stop accepting traces from the process // with the provided PID. After receiving them via ringbuffer, it should // discard them. @@ -103,7 +104,11 @@ type ProcessTracer struct { func (pt *ProcessTracer) AllowPID(pid, ns uint32, svc svc.ID) { for i := range pt.Programs { - pt.Programs[i].AllowPID(pid, ns, svc) + var symTab *gosym.Table + if pt.Goffsets != nil { + symTab = pt.Goffsets.SymTab + } + pt.Programs[i].AllowPID(pid, ns, svc, symTab) } } diff --git a/pkg/internal/export/alloy/traces.go b/pkg/internal/export/alloy/traces.go index c29358fad..168db59a4 100644 --- a/pkg/internal/export/alloy/traces.go +++ b/pkg/internal/export/alloy/traces.go @@ -13,18 +13,18 @@ import ( ) // TracesReceiver creates a terminal node that consumes request.Spans and sends OpenTelemetry traces to the configured consumers. -func TracesReceiver(ctx context.Context, cfg *beyla.TracesReceiverConfig, userAttribSelection attributes.Selection) pipe.FinalProvider[[]request.Span] { +func TracesReceiver(ctx context.Context, cfg *beyla.Config, userAttribSelection attributes.Selection) pipe.FinalProvider[[]request.Span] { return (&tracesReceiver{ctx: ctx, cfg: cfg, attributes: userAttribSelection}).provideLoop } type tracesReceiver struct { ctx context.Context - cfg *beyla.TracesReceiverConfig + cfg *beyla.Config attributes attributes.Selection } func (tr *tracesReceiver) provideLoop() (pipe.FinalFunc[[]request.Span], error) { - if !tr.cfg.Enabled() { + if !tr.cfg.TracesReceiver.Enabled() { return pipe.IgnoreFinal[[]request.Span](), nil } return func(in <-chan []request.Span) { @@ -41,8 +41,8 @@ func (tr *tracesReceiver) provideLoop() (pipe.FinalFunc[[]request.Span], error) continue } - for _, tc := range tr.cfg.Traces { - traces := otel.GenerateTraces(span, traceAttrs) + for _, tc := range tr.cfg.TracesReceiver.Traces { + traces := otel.GenerateTraces(tr.cfg.Traces, span, traceAttrs) err := tc.ConsumeTraces(tr.ctx, traces) if err != nil { slog.Error("error sending trace to consumer", "error", err) diff --git a/pkg/internal/export/debug/debug.go b/pkg/internal/export/debug/debug.go index 8820e3d83..6bea83261 100644 --- a/pkg/internal/export/debug/debug.go +++ b/pkg/internal/export/debug/debug.go @@ -47,6 +47,9 @@ func printFunc() (pipe.FinalFunc[[]request.Span], error) { spans[i].ServiceID.SDKLanguage.String(), traceparent(&spans[i]), ) + if spans[i].ErrorMessage != "" { + fmt.Printf("error_message=%s stacktrace=\n%s\n", spans[i].ErrorMessage, spans[i].ErrorStacktrace) + } } } }, nil diff --git a/pkg/internal/export/otel/traces.go b/pkg/internal/export/otel/traces.go index f0f806bcc..07bc9d142 100644 --- a/pkg/internal/export/otel/traces.go +++ b/pkg/internal/export/otel/traces.go @@ -80,6 +80,9 @@ type TracesConfig struct { // BackOffMaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. BackOffMaxElapsedTime time.Duration `yaml:"backoff_max_elapsed_time" env:"BEYLA_BACKOFF_MAX_ELAPSED_TIME"` + // ReportExceptionEvents enables the reporting of exception events. + ReportExceptionEvents bool `yaml:"report_exception_events" env:"BEYLA_TRACES_REPORT_EXCEPTION_EVENTS"` + ReportersCacheLen int `yaml:"reporters_cache_len" env:"BEYLA_TRACES_REPORT_CACHE_LEN"` // SDKLogLevel works independently from the global LogLevel because it prints GBs of logs in Debug mode @@ -196,7 +199,7 @@ func (tr *tracesOTELReceiver) provideLoop() (pipe.FinalFunc[[]request.Span], err if span.IgnoreSpan == request.IgnoreTraces || !tr.acceptSpan(span) { continue } - traces := GenerateTraces(span, traceAttrs) + traces := GenerateTraces(tr.cfg, span, traceAttrs) err := exp.ConsumeTraces(tr.ctx, traces) if err != nil { slog.Error("error sending trace to consumer", "error", err) @@ -363,7 +366,7 @@ func getRetrySettings(cfg TracesConfig) configretry.BackOffConfig { } // GenerateTraces creates a ptrace.Traces from a request.Span -func GenerateTraces(span *request.Span, userAttrs map[attr.Name]struct{}) ptrace.Traces { +func GenerateTraces(cfg TracesConfig, span *request.Span, userAttrs map[attr.Name]struct{}) ptrace.Traces { t := span.Timings() start := spanStartTime(t) hasSubSpans := t.Start.After(start) @@ -404,6 +407,15 @@ func GenerateTraces(span *request.Span, userAttrs map[attr.Name]struct{}) ptrace m := attrsToMap(attrs) m.CopyTo(s.Attributes()) + // Set error message and stacktrace + if cfg.ReportExceptionEvents && span.ErrorMessage != "" { + e := s.Events().AppendEmpty() + e.SetName(semconv.ExceptionEventName) + e.Attributes().PutStr(string(semconv.ExceptionMessageKey), span.ErrorMessage) + e.Attributes().PutStr(string(semconv.ExceptionTypeKey), "error") + e.Attributes().PutStr(string(semconv.ExceptionStacktraceKey), span.ErrorStacktrace) + } + // Set status code statusCode := codeToStatusCode(request.SpanStatusCode(span)) s.Status().SetCode(statusCode) diff --git a/pkg/internal/export/otel/traces_test.go b/pkg/internal/export/otel/traces_test.go index 3d6dfeb99..8a87f994b 100644 --- a/pkg/internal/export/otel/traces_test.go +++ b/pkg/internal/export/otel/traces_test.go @@ -318,18 +318,23 @@ func TestGenerateTraces(t *testing.T) { spanID, _ := trace.SpanIDFromHex("89cbc1f60aab3b01") traceID, _ := trace.TraceIDFromHex("eae56fbbec9505c102e8aabfc6b5c481") span := &request.Span{ - Type: request.EventTypeHTTP, - RequestStart: start.UnixNano(), - Start: start.Add(time.Second).UnixNano(), - End: start.Add(3 * time.Second).UnixNano(), - Method: "GET", - Route: "/test", - Status: 200, - ParentSpanID: parentSpanID, - TraceID: traceID, - SpanID: spanID, + Type: request.EventTypeHTTP, + RequestStart: start.UnixNano(), + Start: start.Add(time.Second).UnixNano(), + End: start.Add(3 * time.Second).UnixNano(), + Method: "GET", + Route: "/test", + Status: 200, + ParentSpanID: parentSpanID, + TraceID: traceID, + SpanID: spanID, + ErrorMessage: "crash", + ErrorStacktrace: "function\nline", } - traces := GenerateTraces(span, map[attr.Name]struct{}{}) + cfg := TracesConfig{ + ReportExceptionEvents: true, + } + traces := GenerateTraces(cfg, span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -357,6 +362,13 @@ func TestGenerateTraces(t *testing.T) { assert.NotEqual(t, spans.At(0).SpanID().String(), spans.At(1).SpanID().String()) assert.NotEqual(t, spans.At(1).SpanID().String(), spans.At(2).SpanID().String()) + + e := spans.At(2).Events().At(0) + val, _ := e.Attributes().Get(string(semconv.ExceptionMessageKey)) + assert.Equal(t, "crash", val.AsString()) + val, _ = e.Attributes().Get(string(semconv.ExceptionStacktraceKey)) + assert.Equal(t, "function\nline", val.AsString()) + }) t.Run("test with subtraces - ids set bpf layer", func(t *testing.T) { @@ -374,7 +386,8 @@ func TestGenerateTraces(t *testing.T) { SpanID: spanID, TraceID: traceID, } - traces := GenerateTraces(span, map[attr.Name]struct{}{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -410,7 +423,8 @@ func TestGenerateTraces(t *testing.T) { Route: "/test", Status: 200, } - traces := GenerateTraces(span, map[attr.Name]struct{}{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -447,7 +461,8 @@ func TestGenerateTraces(t *testing.T) { SpanID: spanID, TraceID: traceID, } - traces := GenerateTraces(span, map[attr.Name]struct{}{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -472,7 +487,8 @@ func TestGenerateTraces(t *testing.T) { ParentSpanID: parentSpanID, TraceID: traceID, } - traces := GenerateTraces(span, map[attr.Name]struct{}{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -492,7 +508,8 @@ func TestGenerateTraces(t *testing.T) { Method: "GET", Route: "/test", } - traces := GenerateTraces(span, map[attr.Name]struct{}{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -508,7 +525,8 @@ func TestGenerateTraces(t *testing.T) { func TestGenerateTracesAttributes(t *testing.T) { t.Run("test SQL trace generation, no statement", func(t *testing.T) { span := makeSQLRequestSpan("SELECT password FROM credentials WHERE username=\"bill\"") - traces := GenerateTraces(&span, map[attr.Name]struct{}{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -529,7 +547,8 @@ func TestGenerateTracesAttributes(t *testing.T) { t.Run("test SQL trace generation, unknown attribute", func(t *testing.T) { span := makeSQLRequestSpan("SELECT password FROM credentials WHERE username=\"bill\"") - traces := GenerateTraces(&span, map[attr.Name]struct{}{"db.operation.name": {}}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, map[attr.Name]struct{}{"db.operation.name": {}}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -550,7 +569,8 @@ func TestGenerateTracesAttributes(t *testing.T) { t.Run("test SQL trace generation, unknown attribute", func(t *testing.T) { span := makeSQLRequestSpan("SELECT password FROM credentials WHERE username=\"bill\"") - traces := GenerateTraces(&span, map[attr.Name]struct{}{attr.DBQueryText: {}}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, map[attr.Name]struct{}{attr.DBQueryText: {}}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -570,7 +590,8 @@ func TestGenerateTracesAttributes(t *testing.T) { }) t.Run("test Kafka trace generation", func(t *testing.T) { span := request.Span{Type: request.EventTypeKafkaClient, Method: "process", Path: "important-topic", OtherNamespace: "test"} - traces := GenerateTraces(&span, map[attr.Name]struct{}{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, map[attr.Name]struct{}{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -1143,7 +1164,8 @@ func generateTracesForSpans(t *testing.T, tr *tracesOTELReceiver, spans []reques if span.IgnoreSpan == request.IgnoreTraces || !tr.acceptSpan(span) { continue } - res = append(res, GenerateTraces(span, traceAttrs)) + cfg := TracesConfig{} + res = append(res, GenerateTraces(cfg, span, traceAttrs)) } return res diff --git a/pkg/internal/goexec/instructions.go b/pkg/internal/goexec/instructions.go index e96186170..4166888dd 100644 --- a/pkg/internal/goexec/instructions.go +++ b/pkg/internal/goexec/instructions.go @@ -12,9 +12,8 @@ import ( // instrumentationPoints loads the provided executable and looks for the addresses // where the start and return probes must be inserted. -// -//nolint:cyclop -func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncOffsets, error) { +// nolint:cyclop +func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncOffsets, *gosym.Table, error) { ilog := slog.With("component", "goexec.instructions") ilog.Debug("searching for instrumentation points", "functions", funcNames) functions := map[string]struct{}{} @@ -23,12 +22,12 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO } symTab, err := findGoSymbolTable(elfF) if err != nil { - return nil, err + return nil, nil, err } goVersion, _, err := getGoDetails(elfF) if err == nil && !supportedGoVersion(goVersion) { - return nil, fmt.Errorf("unsupported Go version: %v. Minimum supported version is %v", goVersion, minGoVersion) + return nil, nil, fmt.Errorf("unsupported Go version: %v. Minimum supported version is %v", goVersion, minGoVersion) } gosyms := elfF.Section(".gosymtab") @@ -40,7 +39,7 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO if gosyms == nil { allSyms, err = exec.FindExeSymbols(elfF, functions) if err != nil { - return nil, err + return nil, nil, err } } @@ -65,7 +64,7 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO offs, ok, err := findFuncOffset(&f, elfF) if err != nil { - return nil, err + return nil, nil, err } if ok { ilog.Debug("found relevant function for instrumentation", "function", fName, "offsets", offs) @@ -74,7 +73,7 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO } } - return allOffsets, nil + return allOffsets, symTab, nil } func handleStaticSymbol(fName string, allOffsets map[string]FuncOffsets, allSyms map[string]exec.Sym, ilog *slog.Logger) { diff --git a/pkg/internal/goexec/offsets.go b/pkg/internal/goexec/offsets.go index 525c05a4c..07a41a41c 100644 --- a/pkg/internal/goexec/offsets.go +++ b/pkg/internal/goexec/offsets.go @@ -2,15 +2,18 @@ package goexec import ( + "debug/gosym" "fmt" "github.com/grafana/beyla/pkg/internal/exec" + "github.com/grafana/beyla/pkg/internal/export/otel" ) type Offsets struct { // Funcs key: function name - Funcs map[string]FuncOffsets - Field FieldOffsets + Funcs map[string]FuncOffsets + Field FieldOffsets + SymTab *gosym.Table } type FuncOffsets struct { @@ -22,16 +25,22 @@ type FieldOffsets map[string]any // InspectOffsets gets the memory addresses/offsets of the instrumenting function, as well as the required // parameters fields to be read from the eBPF code -func InspectOffsets(execElf *exec.FileInfo, funcs []string) (*Offsets, error) { +func InspectOffsets(cfg *otel.TracesConfig, execElf *exec.FileInfo, funcs []string) (*Offsets, error) { if execElf == nil { return nil, fmt.Errorf("executable not found") } // Analyse executable ELF file and find instrumentation points - found, err := instrumentationPoints(execElf.ELF, funcs) + found, symTab, err := instrumentationPoints(execElf.ELF, funcs) if err != nil { return nil, fmt.Errorf("finding instrumentation points: %w", err) } + // symTab would be used to find the function name from the address when + // capturing Go errors. If the option is disabled, whe don't need to keep + // the symbol table in memory. + if !cfg.ReportExceptionEvents { + symTab = nil + } if len(found) == 0 { return nil, fmt.Errorf("couldn't find any instrumentation point in %s", execElf.CmdExePath) } @@ -43,7 +52,8 @@ func InspectOffsets(execElf *exec.FileInfo, funcs []string) (*Offsets, error) { } return &Offsets{ - Funcs: found, - Field: structFieldOffsets, + Funcs: found, + Field: structFieldOffsets, + SymTab: symTab, }, nil } diff --git a/pkg/internal/goexec/offsets_test.go b/pkg/internal/goexec/offsets_test.go index 5bfa88259..f7823c9e9 100644 --- a/pkg/internal/goexec/offsets_test.go +++ b/pkg/internal/goexec/offsets_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/grafana/beyla/pkg/internal/export/otel" "github.com/grafana/beyla/pkg/internal/testutil" ) @@ -18,7 +19,8 @@ func TestProcessNotFound(t *testing.T) { finish := make(chan struct{}) go func() { defer close(finish) - _, err := InspectOffsets(nil, nil) + cfg := &otel.TracesConfig{} + _, err := InspectOffsets(cfg, nil, nil) require.Error(t, err) }() testutil.ReadChannel(t, finish, 5*time.Second) diff --git a/pkg/internal/pipe/instrumenter.go b/pkg/internal/pipe/instrumenter.go index db16ee878..96a9f152a 100644 --- a/pkg/internal/pipe/instrumenter.go +++ b/pkg/internal/pipe/instrumenter.go @@ -117,7 +117,7 @@ func newGraphBuilder(ctx context.Context, config *beyla.Config, ctxInfo *global. config.Traces.Grafana = &gb.config.Grafana.OTLP pipe.AddFinalProvider(gnb, otelTraces, otel.TracesReceiver(ctx, config.Traces, gb.ctxInfo, config.Attributes.Select)) pipe.AddFinalProvider(gnb, prometheus, prom.PrometheusEndpoint(ctx, gb.ctxInfo, &config.Prometheus, config.Attributes.Select)) - pipe.AddFinalProvider(gnb, alloyTraces, alloy.TracesReceiver(ctx, &config.TracesReceiver, config.Attributes.Select)) + pipe.AddFinalProvider(gnb, alloyTraces, alloy.TracesReceiver(ctx, config, config.Attributes.Select)) pipe.AddFinalProvider(gnb, noop, debug.NoopNode(config.Noop)) pipe.AddFinalProvider(gnb, printer, debug.PrinterNode(config.Printer)) diff --git a/pkg/internal/request/span.go b/pkg/internal/request/span.go index 84361e29c..8206fc672 100644 --- a/pkg/internal/request/span.go +++ b/pkg/internal/request/span.go @@ -63,30 +63,32 @@ type PidInfo struct { // REMINDER: any attribute here must be also added to the functions SpanOTELGetters, // SpanPromGetters and getDefinitions in pkg/internal/export/metric/definitions.go type Span struct { - Type EventType - IgnoreSpan IgnoreMode - Method string - Path string - Route string - Peer string - PeerPort int - Host string - HostPort int - Status int - ContentLength int64 - RequestStart int64 - Start int64 - End int64 - ServiceID svc.ID // TODO: rename to Service or ResourceAttrs - TraceID trace2.TraceID - SpanID trace2.SpanID - ParentSpanID trace2.SpanID - Flags uint8 - Pid PidInfo - PeerName string - HostName string - OtherNamespace string - Statement string + Type EventType + IgnoreSpan IgnoreMode + Method string + Path string + Route string + Peer string + PeerPort int + Host string + HostPort int + Status int + ContentLength int64 + RequestStart int64 + Start int64 + End int64 + ServiceID svc.ID // TODO: rename to Service or ResourceAttrs + TraceID trace2.TraceID + SpanID trace2.SpanID + ParentSpanID trace2.SpanID + Flags uint8 + Pid PidInfo + PeerName string + HostName string + OtherNamespace string + Statement string + ErrorMessage string + ErrorStacktrace string } func (s *Span) Inside(parent *Span) bool { diff --git a/test/integration/components/jaeger/jaeger.go b/test/integration/components/jaeger/jaeger.go index c3dfa19a1..3bf94d148 100644 --- a/test/integration/components/jaeger/jaeger.go +++ b/test/integration/components/jaeger/jaeger.go @@ -28,6 +28,7 @@ type Span struct { Duration int64 `json:"duration"` Tags []Tag `json:"tags"` ProcessID string `json:"processID"` + Logs []Log `json:"logs"` } type Tag struct { @@ -36,6 +37,11 @@ type Tag struct { Value interface{} `json:"value"` } +type Log struct { + Timestamp int64 `json:"timestamp"` + Fields []Tag `json:"fields"` +} + type Reference struct { RefType string `json:"refType"` TraceID string `json:"traceID"` diff --git a/test/integration/components/testserver/std/std.go b/test/integration/components/testserver/std/std.go index e47478704..6179e2f4f 100644 --- a/test/integration/components/testserver/std/std.go +++ b/test/integration/components/testserver/std/std.go @@ -54,6 +54,10 @@ func HTTPHandler(log *slog.Logger, echoPort int) http.HandlerFunc { } else { status = s } + if status == 500 { + echoError(rw) + return + } case arg.Delay: if d, err := time.ParseDuration(v[0]); err != nil { log.Debug("wrong delay value. Ignoring", "error", err) @@ -180,6 +184,12 @@ func echoCall(rw http.ResponseWriter) { rw.WriteHeader(204) } +func echoError(rw http.ResponseWriter) { + _, err := httpClient.Get("htt://pytestserver:8083/error") + slog.Error("error making http request", "err", err) + rw.WriteHeader(500) +} + func Setup(port int) { log := slog.With("component", "std.Server") address := fmt.Sprintf(":%d", port) diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index 082cfc559..38c527fbd 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -49,6 +49,7 @@ services: BEYLA_INTERNAL_METRICS_PROMETHEUS_PORT: 8999 BEYLA_PROCESSES_INTERVAL: "100ms" BEYLA_HOSTNAME: "beyla" + BEYLA_TRACES_REPORT_EXCEPTION_EVENTS: "true" ports: - "8999:8999" # Prometheus scrape port, if enabled via config diff --git a/test/integration/http2go_test.go b/test/integration/http2go_test.go index af8ff3613..2a99d6e0f 100644 --- a/test/integration/http2go_test.go +++ b/test/integration/http2go_test.go @@ -32,7 +32,7 @@ func testREDMetricsForHTTP2Library(t *testing.T, route, svcNs string) { `http_route="` + route + `",` + `url_path="` + route + `"}`) require.NoError(t, err) - // check duration_count has 3 calls and all the arguments + // check duration_count has 1 calls and all the arguments enoughPromResults(t, results) val := totalPromCount(t, results) assert.LessOrEqual(t, 1, val) diff --git a/test/integration/suites_test.go b/test/integration/suites_test.go index c7ef60e22..33491260e 100644 --- a/test/integration/suites_test.go +++ b/test/integration/suites_test.go @@ -130,7 +130,7 @@ func TestSuite_OldestGoVersion(t *testing.T) { require.NoError(t, err) require.NoError(t, compose.Up()) t.Run("RED metrics", testREDMetricsOldHTTP) - t.Run("HTTP traces", testHTTPTraces) + t.Run("HTTP traces", testHTTPTracesNoErrors) t.Run("GRPC traces", testGRPCTraces) t.Run("GRPC RED metrics", testREDMetricsGRPC) t.Run("Internal Prometheus metrics", testInternalPrometheusExport) diff --git a/test/integration/traces_test.go b/test/integration/traces_test.go index bc7c9a215..4ebe4eae9 100644 --- a/test/integration/traces_test.go +++ b/test/integration/traces_test.go @@ -19,14 +19,18 @@ import ( ) func testHTTPTracesNoTraceID(t *testing.T) { - testHTTPTracesCommon(t, false, 200) + testHTTPTracesCommon(t, false, 200, false) } func testHTTPTraces(t *testing.T) { - testHTTPTracesCommon(t, true, 500) + testHTTPTracesCommon(t, true, 500, true) } -func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int) { +func testHTTPTracesNoErrors(t *testing.T) { + testHTTPTracesCommon(t, true, 500, false) +} + +func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int, checkErrors bool) { var traceID string var parentID string @@ -58,7 +62,7 @@ func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int) { traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/" + slug}) require.Len(t, traces, 1) trace = traces[0] - require.Len(t, trace.Spans, 3) // parent - in queue - processing + require.GreaterOrEqual(t, len(trace.Spans), 3) // parent - in queue - processing }, test.Interval(100*time.Millisecond)) // Check the information of the parent span @@ -91,6 +95,20 @@ func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int) { jaeger.Tag{Key: "otel.status_code", Type: "string", Value: "ERROR"}, ) assert.Empty(t, sd, sd.String()) + if checkErrors { + assert.NotEmpty(t, parent.Logs) + eventAttr := parent.Logs[0].Fields + eventType, ok := jaeger.FindIn(eventAttr, "event") + require.True(t, ok) + assert.Equal(t, "exception", eventType.Value) + + errMessage, ok := jaeger.FindIn(eventAttr, "exception.message") + require.True(t, ok) + assert.Equal(t, "unsupported protocol scheme \"htt\"", errMessage.Value) + + _, ok = jaeger.FindIn(eventAttr, "exception.stacktrace") + require.True(t, ok) + } } // Check the information of the "in queue" span