From b3a3181e193f10a89eafa6f0af734b8d52060c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geyslan=20Greg=C3=B3rio?= Date: Tue, 1 Oct 2024 14:08:13 -0300 Subject: [PATCH] wip --- api/v1beta1/diagnostic.pb.go | 147 ++++++++++++++++++---------------- api/v1beta1/diagnostic.proto | 13 +-- pkg/ebpf/c/common/buffer.h | 30 ++++--- pkg/ebpf/perf_count.go | 67 +++++++++++----- pkg/metrics/stats.go | 36 ++++++--- pkg/server/grpc/diagnostic.go | 21 ++--- 6 files changed, 189 insertions(+), 125 deletions(-) diff --git a/api/v1beta1/diagnostic.pb.go b/api/v1beta1/diagnostic.pb.go index 15a8352cffc6..e26a58a48b60 100644 --- a/api/v1beta1/diagnostic.pb.go +++ b/api/v1beta1/diagnostic.pb.go @@ -124,16 +124,17 @@ type GetMetricsResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - EventCount uint64 `protobuf:"varint,1,opt,name=EventCount,proto3" json:"EventCount,omitempty"` - EventsFiltered uint64 `protobuf:"varint,2,opt,name=EventsFiltered,proto3" json:"EventsFiltered,omitempty"` - NetCapCount uint64 `protobuf:"varint,3,opt,name=NetCapCount,proto3" json:"NetCapCount,omitempty"` - BPFLogsCount uint64 `protobuf:"varint,4,opt,name=BPFLogsCount,proto3" json:"BPFLogsCount,omitempty"` - BPFPerfEventWrites uint64 `protobuf:"varint,5,opt,name=BPFPerfEventWrites,proto3" json:"BPFPerfEventWrites,omitempty"` - ErrorCount uint64 `protobuf:"varint,6,opt,name=ErrorCount,proto3" json:"ErrorCount,omitempty"` - LostEvCount uint64 `protobuf:"varint,7,opt,name=LostEvCount,proto3" json:"LostEvCount,omitempty"` - LostWrCount uint64 `protobuf:"varint,8,opt,name=LostWrCount,proto3" json:"LostWrCount,omitempty"` - LostNtCapCount uint64 `protobuf:"varint,9,opt,name=LostNtCapCount,proto3" json:"LostNtCapCount,omitempty"` - LostBPFLogsCount uint64 `protobuf:"varint,10,opt,name=LostBPFLogsCount,proto3" json:"LostBPFLogsCount,omitempty"` + EventCount uint64 `protobuf:"varint,1,opt,name=EventCount,proto3" json:"EventCount,omitempty"` + EventsFiltered uint64 `protobuf:"varint,2,opt,name=EventsFiltered,proto3" json:"EventsFiltered,omitempty"` + NetCapCount uint64 `protobuf:"varint,3,opt,name=NetCapCount,proto3" json:"NetCapCount,omitempty"` + BPFLogsCount uint64 `protobuf:"varint,4,opt,name=BPFLogsCount,proto3" json:"BPFLogsCount,omitempty"` + BPFPerfEventWriteAttempts uint64 `protobuf:"varint,5,opt,name=BPFPerfEventWriteAttempts,proto3" json:"BPFPerfEventWriteAttempts,omitempty"` + BPFPerfEventWriteFailures uint64 `protobuf:"varint,6,opt,name=BPFPerfEventWriteFailures,proto3" json:"BPFPerfEventWriteFailures,omitempty"` + ErrorCount uint64 `protobuf:"varint,7,opt,name=ErrorCount,proto3" json:"ErrorCount,omitempty"` + LostEvCount uint64 `protobuf:"varint,8,opt,name=LostEvCount,proto3" json:"LostEvCount,omitempty"` + LostWrCount uint64 `protobuf:"varint,9,opt,name=LostWrCount,proto3" json:"LostWrCount,omitempty"` + LostNtCapCount uint64 `protobuf:"varint,10,opt,name=LostNtCapCount,proto3" json:"LostNtCapCount,omitempty"` + LostBPFLogsCount uint64 `protobuf:"varint,11,opt,name=LostBPFLogsCount,proto3" json:"LostBPFLogsCount,omitempty"` } func (x *GetMetricsResponse) Reset() { @@ -196,9 +197,16 @@ func (x *GetMetricsResponse) GetBPFLogsCount() uint64 { return 0 } -func (x *GetMetricsResponse) GetBPFPerfEventWrites() uint64 { +func (x *GetMetricsResponse) GetBPFPerfEventWriteAttempts() uint64 { if x != nil { - return x.BPFPerfEventWrites + return x.BPFPerfEventWriteAttempts + } + return 0 +} + +func (x *GetMetricsResponse) GetBPFPerfEventWriteFailures() uint64 { + if x != nil { + return x.BPFPerfEventWriteFailures } return 0 } @@ -415,7 +423,7 @@ var file_api_v1beta1_diagnostic_proto_rawDesc = []byte{ 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x8a, 0x03, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x65, 0x73, 0x74, 0x22, 0xd6, 0x03, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x45, 0x76, @@ -425,60 +433,65 @@ var file_api_v1beta1_diagnostic_proto_rawDesc = []byte{ 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x43, 0x61, 0x70, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x42, 0x50, 0x46, 0x4c, 0x6f, 0x67, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x42, 0x50, 0x46, 0x4c, - 0x6f, 0x67, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x12, 0x42, 0x50, 0x46, 0x50, - 0x65, 0x72, 0x66, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x12, 0x42, 0x50, 0x46, 0x50, 0x65, 0x72, 0x66, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x4c, 0x6f, 0x73, 0x74, - 0x45, 0x76, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x4c, - 0x6f, 0x73, 0x74, 0x45, 0x76, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x4c, 0x6f, - 0x73, 0x74, 0x57, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x0b, 0x4c, 0x6f, 0x73, 0x74, 0x57, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x0e, - 0x4c, 0x6f, 0x73, 0x74, 0x4e, 0x74, 0x43, 0x61, 0x70, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x4c, 0x6f, 0x73, 0x74, 0x4e, 0x74, 0x43, 0x61, 0x70, 0x43, - 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x10, 0x4c, 0x6f, 0x73, 0x74, 0x42, 0x50, 0x46, 0x4c, - 0x6f, 0x67, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, - 0x4c, 0x6f, 0x73, 0x74, 0x42, 0x50, 0x46, 0x4c, 0x6f, 0x67, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x22, 0x47, 0x0a, 0x15, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x05, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, - 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x18, 0x0a, 0x16, 0x43, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, - 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x37, 0x0a, 0x15, 0x47, - 0x65, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, - 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, - 0x72, 0x61, 0x63, 0x65, 0x2a, 0x56, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x12, 0x09, 0x0a, 0x05, 0x44, 0x65, 0x62, 0x75, 0x67, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, - 0x6e, 0x66, 0x6f, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x61, 0x72, 0x6e, 0x10, 0x02, 0x12, - 0x09, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x50, - 0x61, 0x6e, 0x69, 0x63, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x61, 0x6e, 0x69, 0x63, 0x10, - 0x05, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x61, 0x74, 0x61, 0x6c, 0x10, 0x06, 0x32, 0xa7, 0x02, 0x0a, - 0x11, 0x44, 0x69, 0x61, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x12, 0x53, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x12, 0x21, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, - 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, - 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x0e, 0x43, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, 0x2e, 0x74, 0x72, 0x61, 0x63, - 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x26, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, - 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5c, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x53, - 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, 0x24, 0x2e, 0x74, 0x72, 0x61, 0x63, - 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, - 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x25, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x2f, 0x61, 0x71, 0x75, 0x61, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, - 0x2f, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x62, 0x65, - 0x74, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x67, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x3c, 0x0a, 0x19, 0x42, 0x50, 0x46, 0x50, + 0x65, 0x72, 0x66, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x57, 0x72, 0x69, 0x74, 0x65, 0x41, 0x74, 0x74, + 0x65, 0x6d, 0x70, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, 0x42, 0x50, 0x46, + 0x50, 0x65, 0x72, 0x66, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x57, 0x72, 0x69, 0x74, 0x65, 0x41, 0x74, + 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x19, 0x42, 0x50, 0x46, 0x50, 0x65, 0x72, + 0x66, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x57, 0x72, 0x69, 0x74, 0x65, 0x46, 0x61, 0x69, 0x6c, 0x75, + 0x72, 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x19, 0x42, 0x50, 0x46, 0x50, 0x65, + 0x72, 0x66, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x57, 0x72, 0x69, 0x74, 0x65, 0x46, 0x61, 0x69, 0x6c, + 0x75, 0x72, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x4c, 0x6f, 0x73, 0x74, 0x45, 0x76, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x4c, 0x6f, 0x73, 0x74, 0x45, + 0x76, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x4c, 0x6f, 0x73, 0x74, 0x57, 0x72, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x4c, 0x6f, 0x73, + 0x74, 0x57, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x4c, 0x6f, 0x73, 0x74, + 0x4e, 0x74, 0x43, 0x61, 0x70, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0e, 0x4c, 0x6f, 0x73, 0x74, 0x4e, 0x74, 0x43, 0x61, 0x70, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x2a, 0x0a, 0x10, 0x4c, 0x6f, 0x73, 0x74, 0x42, 0x50, 0x46, 0x4c, 0x6f, 0x67, 0x73, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x4c, 0x6f, 0x73, 0x74, + 0x42, 0x50, 0x46, 0x4c, 0x6f, 0x67, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x47, 0x0a, 0x15, + 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, + 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, + 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x18, 0x0a, 0x16, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x4c, + 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x16, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x37, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x74, + 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1e, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, + 0x2a, 0x56, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x09, 0x0a, 0x05, + 0x44, 0x65, 0x62, 0x75, 0x67, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x10, + 0x01, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x61, 0x72, 0x6e, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x50, 0x61, 0x6e, 0x69, 0x63, + 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x61, 0x6e, 0x69, 0x63, 0x10, 0x05, 0x12, 0x09, 0x0a, + 0x05, 0x46, 0x61, 0x74, 0x61, 0x6c, 0x10, 0x06, 0x32, 0xa7, 0x02, 0x0a, 0x11, 0x44, 0x69, 0x61, + 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x53, + 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x21, 0x2e, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x22, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x0e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x25, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, + 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x43, 0x68, + 0x61, 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5c, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, + 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, 0x24, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, + 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x74, 0x72, + 0x61, 0x63, 0x65, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, + 0x53, 0x74, 0x61, 0x63, 0x6b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x2f, + 0x61, 0x71, 0x75, 0x61, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x61, + 0x63, 0x65, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/v1beta1/diagnostic.proto b/api/v1beta1/diagnostic.proto index 78af1d2bdc14..78dea97cdf14 100644 --- a/api/v1beta1/diagnostic.proto +++ b/api/v1beta1/diagnostic.proto @@ -12,12 +12,13 @@ message GetMetricsResponse { uint64 EventsFiltered = 2; uint64 NetCapCount = 3; uint64 BPFLogsCount = 4; - uint64 BPFPerfEventWrites = 5; - uint64 ErrorCount = 6; - uint64 LostEvCount = 7; - uint64 LostWrCount = 8; - uint64 LostNtCapCount = 9; - uint64 LostBPFLogsCount = 10; + uint64 BPFPerfEventWriteAttempts = 5; + uint64 BPFPerfEventWriteFailures = 6; + uint64 ErrorCount = 7; + uint64 LostEvCount = 8; + uint64 LostWrCount = 9; + uint64 LostNtCapCount = 10; + uint64 LostBPFLogsCount = 11; } enum LogLevel { diff --git a/pkg/ebpf/c/common/buffer.h b/pkg/ebpf/c/common/buffer.h index 9bcef571e9a3..f044cd221619 100644 --- a/pkg/ebpf/c/common/buffer.h +++ b/pkg/ebpf/c/common/buffer.h @@ -459,12 +459,18 @@ statfunc int save_args_to_submit_buf(event_data_t *event, args_t *args) } #ifdef DEBUG -struct event_counts { + +typedef struct { + u64 attempts; + u64 failures; +} event_stats_values_t; + +struct events_stats { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, MAX_EVENT_ID); - __type(key, u32); // eventid - __type(value, u64); // count -} event_counts SEC(".maps"); + __type(key, u32); // eventid + __type(value, event_stats_values_t); +} events_stats SEC(".maps"); #endif statfunc int events_perf_submit(program_data_t *p, long ret) @@ -493,14 +499,20 @@ statfunc int events_perf_submit(program_data_t *p, long ret) : : [size] "r"(size), [max_size] "i"(MAX_EVENT_SIZE)); + u64 perf_ret = bpf_perf_event_output(p->ctx, &events, BPF_F_CURRENT_CPU, p->event, size); + #ifdef DEBUG - // increment event count before event submission attempt - u64 *event_count = bpf_map_lookup_elem(&event_counts, &p->event->context.eventid); - if (event_count) - __sync_fetch_and_add(event_count, 1); + // update event stats + event_stats_values_t *evt_stat = bpf_map_lookup_elem(&events_stats, &p->event->context.eventid); + if (unlikely(evt_stat == NULL)) + return perf_ret; + + __sync_fetch_and_add(&evt_stat->attempts, 1); + if (perf_ret < 0) + __sync_fetch_and_add(&evt_stat->failures, 1); #endif - return bpf_perf_event_output(p->ctx, &events, BPF_F_CURRENT_CPU, p->event, size); + return perf_ret; } statfunc int signal_perf_submit(void *ctx, controlplane_signal_t *sig) diff --git a/pkg/ebpf/perf_count.go b/pkg/ebpf/perf_count.go index 8cc9f1326e79..06ccfb8b7d9a 100644 --- a/pkg/ebpf/perf_count.go +++ b/pkg/ebpf/perf_count.go @@ -11,29 +11,35 @@ import ( "github.com/aquasecurity/tracee/pkg/logger" ) +type eventStatsValues struct { + attempts uint64 + failures uint64 +} + // countPerfEventWrites counts the number of times each event is attempted // to be written to the perf buffer. func (t *Tracee) countPerfEventWrites(ctx context.Context) { logger.Debugw("Starting countPerfEventWrites goroutine") defer logger.Debugw("Stopped countPerfEventWrites goroutine") - evtsCountsBPFMap, err := t.bpfModule.GetMap("event_counts") + evtsCountsBPFMap, err := t.bpfModule.GetMap("events_stats") if err != nil { - logger.Errorw("Failed to get event_counts map", "error", err) + logger.Errorw("Failed to get events_stats map", "error", err) return } + evtStatZero := eventStatsValues{} for _, id := range t.policyManager.EventsSelected() { key := uint32(id) - value := uint64(0) - err := evtsCountsBPFMap.Update(unsafe.Pointer(&key), unsafe.Pointer(&value)) + err := evtsCountsBPFMap.Update(unsafe.Pointer(&key), unsafe.Pointer(&evtStatZero)) if err != nil { - logger.Errorw("Failed to update event_counts map", "error", err) + logger.Errorw("Failed to update events_stats map", "error", err) } } - total := counter.NewCounter(0) - evtsCounts := make(map[uint32]uint64) + totalAttempts := counter.NewCounter(0) + totalFailures := counter.NewCounter(0) + evtsCounts := make(map[uint32]eventStatsValues) ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -42,36 +48,55 @@ func (t *Tracee) countPerfEventWrites(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: + // Get the counts of each event from the BPF map into a hashmap iter := evtsCountsBPFMap.Iterator() for iter.Next() { key := binary.LittleEndian.Uint32(iter.Key()) value, err := evtsCountsBPFMap.GetValue(unsafe.Pointer(&key)) if err != nil { - logger.Errorw("Failed to get value from event_counts map", "error", err) + logger.Errorw("Failed to get value from events_stats map", "error", err) continue } - evtsCounts[key] = binary.LittleEndian.Uint64(value) + evtsCounts[key] = eventStatsValues{ + attempts: binary.LittleEndian.Uint64(value[0:8]), + failures: binary.LittleEndian.Uint64(value[8:16]), + } } - total.Set(0) + // Get the counts of each event from the hashmap into a slice (key value pairs) + // and calculate the total count + keyValsAttempts := make([]interface{}, 0, len(evtsCounts)*2+1) + keyValsFailures := make([]interface{}, 0, len(evtsCounts)*2+1) + totalAttempts.Set(0) + totalFailures.Set(0) for k, v := range evtsCounts { - if v == 0 { - continue - } - err := total.Increment(v) + keyValsAttempts = append(keyValsAttempts, + events.Core.GetDefinitionByID(events.ID(k)).GetName(), + v.attempts, + ) + keyValsFailures = append(keyValsFailures, + events.Core.GetDefinitionByID(events.ID(k)).GetName(), + v.failures, + ) + + err := totalAttempts.Increment(v.attempts) if err != nil { logger.Errorw("Failed to increment total counter", "error", err) } - - logger.Debugw("Event sending attempts", - "event", events.Core.GetDefinitionByID(events.ID(k)).GetName(), - "count", v, - ) + err = totalFailures.Increment(v.failures) + if err != nil { + logger.Errorw("Failed to increment total failures counter", "error", err) + } } - logger.Debugw("Event sending attempts", "total", total.Get()) - t.stats.BPFPerfEventWrites.Set(total.Get()) + // Log the counts + keyValsAttempts = append(keyValsAttempts, "total", totalAttempts.Get()) + logger.Infow("Event sending attempts", keyValsAttempts...) + keyValsFailures = append(keyValsFailures, "total", totalFailures.Get()) + logger.Infow("Event sending failures", keyValsFailures...) + t.stats.BPFPerfEventWriteAttempts.Set(totalAttempts.Get()) + t.stats.BPFPerfEventWriteFailures.Set(totalFailures.Get()) } } } diff --git a/pkg/metrics/stats.go b/pkg/metrics/stats.go index 45f86cd09ef6..8498e38397ea 100644 --- a/pkg/metrics/stats.go +++ b/pkg/metrics/stats.go @@ -9,16 +9,17 @@ import ( // When updating this struct, please make sure to update the relevant exporting functions type Stats struct { - EventCount counter.Counter - EventsFiltered counter.Counter - NetCapCount counter.Counter // network capture events - BPFLogsCount counter.Counter - BPFPerfEventWrites counter.Counter // calls to write to the event perf buffer - ErrorCount counter.Counter - LostEvCount counter.Counter - LostWrCount counter.Counter - LostNtCapCount counter.Counter // lost network capture events - LostBPFLogsCount counter.Counter + EventCount counter.Counter + EventsFiltered counter.Counter + NetCapCount counter.Counter // network capture events + BPFLogsCount counter.Counter + BPFPerfEventWriteAttempts counter.Counter // calls to write to the event perf buffer + BPFPerfEventWriteFailures counter.Counter // failed calls to write to the event perf buffer + ErrorCount counter.Counter + LostEvCount counter.Counter + LostWrCount counter.Counter + LostNtCapCount counter.Counter // lost network capture events + LostBPFLogsCount counter.Counter } // Register Stats to prometheus metrics exporter @@ -65,9 +66,20 @@ func (stats *Stats) RegisterPrometheus() error { err = prometheus.Register(prometheus.NewCounterFunc(prometheus.CounterOpts{ Namespace: "tracee_ebpf", - Name: "bpf_perf_event_writes_total", + Name: "bpf_perf_event_write_attempts_total", Help: "total number of calls to write to the event perf buffer", - }, func() float64 { return float64(stats.BPFPerfEventWrites.Get()) })) + }, func() float64 { return float64(stats.BPFPerfEventWriteAttempts.Get()) })) + + if err != nil { + return errfmt.WrapError(err) + } + + // This must be similar to LostWrCount but is updated in a different manner + err = prometheus.Register(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "tracee_ebpf", + Name: "bpf_perf_event_write_failures_total", + Help: "total number of failed calls to write to the event perf buffer", + }, func() float64 { return float64(stats.BPFPerfEventWriteFailures.Get()) })) if err != nil { return errfmt.WrapError(err) diff --git a/pkg/server/grpc/diagnostic.go b/pkg/server/grpc/diagnostic.go index 4f703cb81bca..93ac65194b01 100644 --- a/pkg/server/grpc/diagnostic.go +++ b/pkg/server/grpc/diagnostic.go @@ -17,16 +17,17 @@ type DiagnosticService struct { func (s *DiagnosticService) GetMetrics(ctx context.Context, in *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { stats := s.tracee.Stats() metrics := &pb.GetMetricsResponse{ - EventCount: stats.EventCount.Get(), - EventsFiltered: stats.EventsFiltered.Get(), - NetCapCount: stats.NetCapCount.Get(), - BPFLogsCount: stats.BPFLogsCount.Get(), - BPFPerfEventWrites: stats.BPFPerfEventWrites.Get(), // only available in debug build - ErrorCount: stats.ErrorCount.Get(), - LostEvCount: stats.LostEvCount.Get(), - LostWrCount: stats.LostWrCount.Get(), - LostNtCapCount: stats.LostNtCapCount.Get(), - LostBPFLogsCount: stats.LostBPFLogsCount.Get(), + EventCount: stats.EventCount.Get(), + EventsFiltered: stats.EventsFiltered.Get(), + NetCapCount: stats.NetCapCount.Get(), + BPFLogsCount: stats.BPFLogsCount.Get(), + BPFPerfEventWriteAttempts: stats.BPFPerfEventWriteAttempts.Get(), // only available in debug build + BPFPerfEventWriteFailures: stats.BPFPerfEventWriteFailures.Get(), // only available in debug build + ErrorCount: stats.ErrorCount.Get(), + LostEvCount: stats.LostEvCount.Get(), + LostWrCount: stats.LostWrCount.Get(), + LostNtCapCount: stats.LostNtCapCount.Get(), + LostBPFLogsCount: stats.LostBPFLogsCount.Get(), } return metrics, nil