Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix SQL detection #1427

Merged
merged 7 commits into from
Dec 3, 2024
Merged

Fix SQL detection #1427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1397b0b
more precise sql detection
grcevski Dec 2, 2024
8c8f6f2
fix bugs
grcevski Dec 2, 2024
d3ed7f8
Add mysql test, fix bugs with tcp SSL
grcevski Dec 2, 2024
1f54ec7
Add support for prepared mysql statements, integration tests
grcevski Dec 2, 2024
4024a4e
Add support for mysql, postgres
grcevski Dec 2, 2024
27c897e
docs for the new option
grcevski Dec 2, 2024
ac119b9
test fixes
grcevski Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions bpf/http_ssl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,11 +173,10 @@ int BPF_UPROBE(uprobe_ssl_write, void *ssl, const void *buf, int num) {
ssl_args_t args = {};
args.buf = (u64)buf;
args.ssl = (u64)ssl;
args.len_ptr = num;

bpf_map_update_elem(&active_ssl_write_args, &id, &args, BPF_ANY);

// must be last in the function, doesn't return
handle_ssl_buf(ctx, id, &args, num, TCP_SEND);
return 0;
}

Expand All @@ -189,9 +188,17 @@ int BPF_URETPROBE(uretprobe_ssl_write, int ret) {
return 0;
}

bpf_dbg_printk("=== uretprobe SSL_write id=%d ===", id);
ssl_args_t *args = bpf_map_lookup_elem(&active_ssl_write_args, &id);

bpf_dbg_printk("=== uretprobe SSL_write id=%d args %llx ===", id, args);

bpf_map_delete_elem(&active_ssl_write_args, &id);
if (args) {
ssl_args_t saved = {};
__builtin_memcpy(&saved, args, sizeof(ssl_args_t));
bpf_map_delete_elem(&active_ssl_write_args, &id);
// must be last in the function, doesn't return
handle_ssl_buf(ctx, id, &saved, saved.len_ptr, TCP_SEND);
}

return 0;
}
Expand All @@ -209,12 +216,10 @@ int BPF_UPROBE(uprobe_ssl_write_ex, void *ssl, const void *buf, int num, size_t
ssl_args_t args = {};
args.buf = (u64)buf;
args.ssl = (u64)ssl;
args.len_ptr = num;

bpf_map_update_elem(&active_ssl_write_args, &id, &args, BPF_ANY);

// must be last in the function, doesn't return
handle_ssl_buf(ctx, id, &args, num, TCP_SEND);

return 0;
}

Expand All @@ -226,9 +231,17 @@ int BPF_URETPROBE(uretprobe_ssl_write_ex, int ret) {
return 0;
}

bpf_dbg_printk("=== uretprobe SSL_write_ex id=%d ===", id);
ssl_args_t *args = bpf_map_lookup_elem(&active_ssl_write_args, &id);

bpf_dbg_printk("=== uretprobe SSL_write_ex id=%d args %llx ===", id, args);

bpf_map_delete_elem(&active_ssl_write_args, &id);
if (args) {
ssl_args_t saved = {};
__builtin_memcpy(&saved, args, sizeof(ssl_args_t));
bpf_map_delete_elem(&active_ssl_write_args, &id);
// must be last in the function, doesn't return
handle_ssl_buf(ctx, id, &saved, saved.len_ptr, TCP_SEND);
}

return 0;
}
Expand Down
21 changes: 13 additions & 8 deletions bpf/http_ssl_defs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,15 +187,14 @@ handle_ssl_buf(void *ctx, u64 id, ssl_args_t *args, int bytes_len, u8 direction)
}

if (conn) {
// bpf_dbg_printk("conn pid %d", conn.pid);
// dbg_print_http_connection_info(&conn->p_conn.conn);
bpf_dbg_printk("SSL conn");
dbg_print_http_connection_info(&conn->p_conn.conn);

// unsigned char buf[48];
// bpf_probe_read(buf, 48, (void *)args->buf);
// for (int i=0; i < 48; i++) {
// bpf_dbg_printk("%x ", buf[i]);
// }
bpf_map_update_elem(&active_ssl_connections, &conn->p_conn, &ssl_ptr, BPF_ANY);

// We should attempt to clean up the server trace immediately. The cleanup information
// is keyed of the *ssl, so when it's delayed we might have different *ssl on the same
Expand All @@ -215,7 +214,11 @@ handle_ssl_buf(void *ctx, u64 id, ssl_args_t *args, int bytes_len, u8 direction)
}
}

static __always_inline void *is_ssl_connection(u64 id) {
static __always_inline void set_active_ssl_connection(pid_connection_info_t *conn, void *ssl) {
bpf_map_update_elem(&active_ssl_connections, conn, &ssl, BPF_ANY);
}

static __always_inline void *is_ssl_connection(u64 id, pid_connection_info_t *conn) {
void *ssl = 0;
// Checks if it's sandwitched between active SSL handshake, read or write uprobe/uretprobe
void **s = bpf_map_lookup_elem(&active_ssl_handshakes, &id);
Expand All @@ -231,11 +234,13 @@ static __always_inline void *is_ssl_connection(u64 id) {
}
}

return ssl;
}
if (!ssl) {
return bpf_map_lookup_elem(&active_ssl_connections, conn);
} else {
set_active_ssl_connection(conn, ssl);
}

static __always_inline void *is_active_ssl(pid_connection_info_t *conn) {
return bpf_map_lookup_elem(&active_ssl_connections, conn);
return ssl;
}

#endif
109 changes: 49 additions & 60 deletions bpf/k_tracer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,55 +282,50 @@ int BPF_KPROBE(kprobe_tcp_sendmsg, struct sock *sk, struct msghdr *msg, size_t s
sort_connection_info(&s_args.p_conn.conn);
s_args.p_conn.pid = pid_from_pid_tgid(id);

void *ssl = is_ssl_connection(id);
void *ssl = is_ssl_connection(id, &s_args.p_conn);
if (size > 0) {
if (!ssl) {
void *active_ssl = is_active_ssl(&s_args.p_conn);
if (!active_ssl) {
u8 *buf = iovec_memory();
if (buf) {
size = read_msghdr_buf(msg, buf, size);
// If a sock_msg program is installed, this kprobe will fail to
// read anything, because the data is in bvec physical pages. However,
// the sock_msg will setup a buffer for us if this is the case. We
// look up this buffer and use it instead of what we'd get from
// calling read_msghdr_buf.
if (!size) {
msg_buffer_t *m_buf = bpf_map_lookup_elem(&msg_buffers, &e_key);
bpf_dbg_printk("No size, m_buf[%llx]", m_buf);
if (m_buf) {
buf = m_buf->buf;
// The buffer setup for us by a sock_msg program is always the
// full buffer, but when we extend a packet to be able to inject
// a Traceparent field, it will actually be split in 3 chunks:
// [before the injected header],[70 bytes for 'Traceparent...'],[the rest].
// We don't want the handle_buf_with_connection logic to run more than
// once on the same data, so if we find a buf we send all of it to the
// handle_buf_with_connection logic and then mark it as seen by making
// m_buf->pos be the size of the buffer.
if (!m_buf->pos) {
size = sizeof(m_buf->buf);
m_buf->pos = size;
bpf_dbg_printk("msg_buffer: size %d, buf[%s]", size, buf);
} else {
size = 0;
}
u8 *buf = iovec_memory();
if (buf) {
size = read_msghdr_buf(msg, buf, size);
// If a sock_msg program is installed, this kprobe will fail to
// read anything, because the data is in bvec physical pages. However,
// the sock_msg will setup a buffer for us if this is the case. We
// look up this buffer and use it instead of what we'd get from
// calling read_msghdr_buf.
if (!size) {
msg_buffer_t *m_buf = bpf_map_lookup_elem(&msg_buffers, &e_key);
bpf_dbg_printk("No size, m_buf[%llx]", m_buf);
if (m_buf) {
buf = m_buf->buf;
// The buffer setup for us by a sock_msg program is always the
// full buffer, but when we extend a packet to be able to inject
// a Traceparent field, it will actually be split in 3 chunks:
// [before the injected header],[70 bytes for 'Traceparent...'],[the rest].
// We don't want the handle_buf_with_connection logic to run more than
// once on the same data, so if we find a buf we send all of it to the
// handle_buf_with_connection logic and then mark it as seen by making
// m_buf->pos be the size of the buffer.
if (!m_buf->pos) {
size = sizeof(m_buf->buf);
m_buf->pos = size;
bpf_dbg_printk("msg_buffer: size %d, buf[%s]", size, buf);
} else {
size = 0;
}
}
if (size) {
u64 sock_p = (u64)sk;
bpf_map_update_elem(&active_send_args, &id, &s_args, BPF_ANY);
bpf_map_update_elem(&active_send_sock_args, &sock_p, &s_args, BPF_ANY);

// Logically last for !ssl.
handle_buf_with_connection(
ctx, &s_args.p_conn, buf, size, NO_SSL, TCP_SEND, orig_dport);
} else {
bpf_dbg_printk("can't find iovec ptr in msghdr, not tracking sendmsg");
}
}
} else {
bpf_dbg_printk("tcp_sendmsg for identified SSL connection, ignoring...");
if (size) {
u64 sock_p = (u64)sk;
bpf_map_update_elem(&active_send_args, &id, &s_args, BPF_ANY);
bpf_map_update_elem(&active_send_sock_args, &sock_p, &s_args, BPF_ANY);

// Logically last for !ssl.
handle_buf_with_connection(
ctx, &s_args.p_conn, buf, size, NO_SSL, TCP_SEND, orig_dport);
} else {
bpf_dbg_printk("can't find iovec ptr in msghdr, not tracking sendmsg");
}
}
} else {
bpf_dbg_printk("tcp_sendmsg for identified SSL connection, ignoring...");
Expand Down Expand Up @@ -487,24 +482,19 @@ static __always_inline int return_recvmsg(void *ctx, u64 id, int copied_len) {
sort_connection_info(&info.conn);
info.pid = pid_from_pid_tgid(id);

void *ssl = is_ssl_connection(id);
void *ssl = is_ssl_connection(id, &info);

if (!ssl) {
void *active_ssl = is_active_ssl(&info);
if (!active_ssl) {
u8 *buf = iovec_memory();
if (buf) {
copied_len = read_iovec_ctx(iov_ctx, buf, copied_len);
if (copied_len) {
// doesn't return must be logically last statement
handle_buf_with_connection(
ctx, &info, buf, copied_len, NO_SSL, TCP_RECV, orig_dport);
} else {
bpf_dbg_printk("Not copied anything");
}
u8 *buf = iovec_memory();
if (buf) {
copied_len = read_iovec_ctx(iov_ctx, buf, copied_len);
if (copied_len) {
// doesn't return must be logically last statement
handle_buf_with_connection(
ctx, &info, buf, copied_len, NO_SSL, TCP_RECV, orig_dport);
} else {
bpf_dbg_printk("Not copied anything");
}
} else {
bpf_dbg_printk("tcp_recvmsg for an identified SSL connection, ignoring...");
}
} else {
bpf_dbg_printk("tcp_recvmsg for an identified SSL connection, ignoring...");
Expand Down Expand Up @@ -680,7 +670,6 @@ int BPF_KPROBE(kprobe_sys_exit, int status) {
if (s_args) {
bpf_dbg_printk("Checking if we need to finish the request per thread id");
finish_possible_delayed_http_request(&s_args->p_conn);
bpf_map_delete_elem(&active_ssl_connections, &s_args->p_conn);
}

bpf_map_delete_elem(&clone_map, &task.p_key);
Expand Down
31 changes: 20 additions & 11 deletions bpf/protocol_tcp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ static __always_inline void handle_unknown_tcp_connection(pid_connection_info_t
u8 ssl,
u16 orig_dport) {
tcp_req_t *existing = bpf_map_lookup_elem(&ongoing_tcp_req, pid_conn);
if (existing) {
if (existing->direction == direction && existing->end_monotime_ns != 0) {
bpf_map_delete_elem(&ongoing_tcp_req, pid_conn);
existing = 0;
}
}
if (!existing) {
tcp_req_t *req = empty_tcp_req();
if (req) {
Expand All @@ -100,6 +106,8 @@ static __always_inline void handle_unknown_tcp_connection(pid_connection_info_t
req->ssl = ssl;
req->direction = direction;
req->start_monotime_ns = bpf_ktime_get_ns();
req->end_monotime_ns = 0;
req->resp_len = 0;
req->len = bytes_len;
task_pid(&req->pid);
bpf_probe_read(req->buf, K_TCP_MAX_LEN, u_buf);
Expand All @@ -113,18 +121,19 @@ static __always_inline void handle_unknown_tcp_connection(pid_connection_info_t
bpf_map_update_elem(&ongoing_tcp_req, pid_conn, req, BPF_ANY);
}
} else if (existing->direction != direction) {
existing->end_monotime_ns = bpf_ktime_get_ns();
existing->resp_len = bytes_len;
tcp_req_t *trace = bpf_ringbuf_reserve(&events, sizeof(tcp_req_t), 0);
if (trace) {
bpf_dbg_printk(
"Sending TCP trace %lx, response length %d", existing, existing->resp_len);

__builtin_memcpy(trace, existing, sizeof(tcp_req_t));
bpf_probe_read(trace->rbuf, K_TCP_RES_LEN, u_buf);
bpf_ringbuf_submit(trace, get_flags());
if (existing->end_monotime_ns == 0) {
existing->end_monotime_ns = bpf_ktime_get_ns();
existing->resp_len = bytes_len;
tcp_req_t *trace = bpf_ringbuf_reserve(&events, sizeof(tcp_req_t), 0);
if (trace) {
bpf_dbg_printk(
"Sending TCP trace %lx, response length %d", existing, existing->resp_len);

__builtin_memcpy(trace, existing, sizeof(tcp_req_t));
bpf_probe_read(trace->rbuf, K_TCP_RES_LEN, u_buf);
bpf_ringbuf_submit(trace, get_flags());
}
}
bpf_map_delete_elem(&ongoing_tcp_req, pid_conn);
} else if (existing->len > 0 && existing->len < (K_TCP_MAX_LEN / 2)) {
// Attempt to append one more packet. I couldn't convince the verifier
// to use a variable (K_TCP_MAX_LEN-existing->len). If needed we may need
Expand Down
33 changes: 15 additions & 18 deletions bpf/tc_sock.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,26 +193,23 @@ static __always_inline u8 protocol_detector(struct sk_msg_md *msg,
sort_connection_info(&s_args.p_conn.conn);
s_args.p_conn.pid = pid_from_pid_tgid(id);

void *ssl = is_ssl_connection(id);
void *ssl = is_ssl_connection(id, &s_args.p_conn);
if (s_args.size > 0) {
if (!ssl) {
void *active_ssl = is_active_ssl(&s_args.p_conn);
if (!active_ssl) {
msg_buffer_t msg_buf = {
.pos = 0,
};
bpf_probe_read_kernel(msg_buf.buf, FULL_BUF_SIZE, msg->data);
// We setup any call that looks like HTTP request to be extended.
// This must match exactly to what the decision will be for
// the kprobe program on tcp_sendmsg, which sets up the
// outgoing_trace_map data used by Traffic Control to write the
// actual 'Traceparent:...' string.
if (is_http_request_buf((const unsigned char *)msg_buf.buf)) {
bpf_dbg_printk("Setting up request to be extended");
bpf_map_update_elem(&msg_buffers, &e_key, &msg_buf, BPF_ANY);

return 1;
}
msg_buffer_t msg_buf = {
.pos = 0,
};
bpf_probe_read_kernel(msg_buf.buf, FULL_BUF_SIZE, msg->data);
// We setup any call that looks like HTTP request to be extended.
// This must match exactly to what the decision will be for
// the kprobe program on tcp_sendmsg, which sets up the
// outgoing_trace_map data used by Traffic Control to write the
// actual 'Traceparent:...' string.
if (is_http_request_buf((const unsigned char *)msg_buf.buf)) {
bpf_dbg_printk("Setting up request to be extended");
bpf_map_update_elem(&msg_buffers, &e_key, &msg_buf, BPF_ANY);

return 1;
}
}
}
Expand Down
16 changes: 15 additions & 1 deletion docs/sources/configure/options.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,8 @@ generation of Beyla metrics.

Configures the time interval after which an HTTP request is considered as a timeout.
This option allows Beyla to report HTTP transactions which timeout and never return.
To disable the automatic HTTP request timeout feature, set this option to zero, i.e. "0ms".
To disable the automatic HTTP request timeout feature, set this option to zero,
that is "0ms".

| YAML | Environment variable | Type | Default |
| ----------------------- | ---------------------------------- | -------- | ------- |
Expand All @@ -514,6 +515,19 @@ Configures the HTTP tracer heuristic to send telemetry events as soon as a respo
Setting this option reduces the accuracy of timings for requests with large responses, however,
in high request volume scenarios this option will reduce the number of dropped trace events.

| YAML | Environment variable | Type | Default |
| ----------------------- | ---------------------------------- | -------- | ------- |
| `heuristic_sql_detect` | `BEYLA_HEURISTIC_SQL_DETECT` | boolean | (false) |

By default, Beyla detects various SQL client requests through detection of their
particular binary protocol format. However, oftentimes SQL database clients send their
queries in a format where Beyla can detect the query statement without knowing
the exact binary protocol. If you are using a database technology not directly supported
by Beyla, you can enable this option to get database client telemetry. The option is
not enabled by default, because it can create false positives, for example, an application
sending SQL text for logging purposes through a TCP connection. Currently supported
protocols where this option isn't needed are the Postgres and MySQL binary protocols.

## Configuration of metrics and traces attributes

Grafana Beyla allows configuring how some attributes for metrics and traces
Expand Down
4 changes: 4 additions & 0 deletions pkg/config/ebpf_tracer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,8 @@ type EPPFTracer struct {

// Optimises for getting requests information immediately when request response is seen
HighRequestVolume bool `yaml:"high_request_volume" env:"BEYLA_BPF_HIGH_REQUEST_VOLUME"`

// Enables the heuristic based detection of SQL requests. This can be used to detect
// talking to databases other than the ones we recognize in Beyla, like Postgres and MySQL
HeuristicSQLDetect bool `yaml:"heuristic_sql_detect" env:"BEYLA_HEURISTIC_SQL_DETECT"`
}
2 changes: 1 addition & 1 deletion pkg/export/otel/traces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ func traceAttributes(span *request.Span, optionalAttrs map[attr.Name]struct{}) [
attrs = []attribute.KeyValue{
request.ServerAddr(request.HostAsServer(span)),
request.ServerPort(span.HostPort),
semconv.DBSystemOtherSQL, // We can distinguish in the future for MySQL, Postgres etc
span.DBSystem(), // We can distinguish in the future for MySQL, Postgres etc
}
if _, ok := optionalAttrs[attr.DBQueryText]; ok {
attrs = append(attrs, request.DBQueryText(span.Statement))
Expand Down
Loading
Loading