diff --git a/bpf/http_ssl.h b/bpf/http_ssl.h index 10bec76ae..827a4f6d7 100644 --- a/bpf/http_ssl.h +++ b/bpf/http_ssl.h @@ -173,11 +173,10 @@ int BPF_UPROBE(uprobe_ssl_write, void *ssl, const void *buf, int num) { ssl_args_t args = {}; args.buf = (u64)buf; args.ssl = (u64)ssl; + args.len_ptr = num; bpf_map_update_elem(&active_ssl_write_args, &id, &args, BPF_ANY); - // must be last in the function, doesn't return - handle_ssl_buf(ctx, id, &args, num, TCP_SEND); return 0; } @@ -189,9 +188,17 @@ int BPF_URETPROBE(uretprobe_ssl_write, int ret) { return 0; } - bpf_dbg_printk("=== uretprobe SSL_write id=%d ===", id); + ssl_args_t *args = bpf_map_lookup_elem(&active_ssl_write_args, &id); + + bpf_dbg_printk("=== uretprobe SSL_write id=%d args %llx ===", id, args); - bpf_map_delete_elem(&active_ssl_write_args, &id); + if (args) { + ssl_args_t saved = {}; + __builtin_memcpy(&saved, args, sizeof(ssl_args_t)); + bpf_map_delete_elem(&active_ssl_write_args, &id); + // must be last in the function, doesn't return + handle_ssl_buf(ctx, id, &saved, saved.len_ptr, TCP_SEND); + } return 0; } @@ -209,12 +216,10 @@ int BPF_UPROBE(uprobe_ssl_write_ex, void *ssl, const void *buf, int num, size_t ssl_args_t args = {}; args.buf = (u64)buf; args.ssl = (u64)ssl; + args.len_ptr = num; bpf_map_update_elem(&active_ssl_write_args, &id, &args, BPF_ANY); - // must be last in the function, doesn't return - handle_ssl_buf(ctx, id, &args, num, TCP_SEND); - return 0; } @@ -226,9 +231,17 @@ int BPF_URETPROBE(uretprobe_ssl_write_ex, int ret) { return 0; } - bpf_dbg_printk("=== uretprobe SSL_write_ex id=%d ===", id); + ssl_args_t *args = bpf_map_lookup_elem(&active_ssl_write_args, &id); + + bpf_dbg_printk("=== uretprobe SSL_write_ex id=%d args %llx ===", id, args); - bpf_map_delete_elem(&active_ssl_write_args, &id); + if (args) { + ssl_args_t saved = {}; + __builtin_memcpy(&saved, args, sizeof(ssl_args_t)); + bpf_map_delete_elem(&active_ssl_write_args, &id); + // must be last in the function, doesn't return + handle_ssl_buf(ctx, id, &saved, saved.len_ptr, TCP_SEND); + } return 0; } diff --git a/bpf/http_ssl_defs.h b/bpf/http_ssl_defs.h index aaa181cec..7500cc037 100644 --- a/bpf/http_ssl_defs.h +++ b/bpf/http_ssl_defs.h @@ -187,15 +187,14 @@ handle_ssl_buf(void *ctx, u64 id, ssl_args_t *args, int bytes_len, u8 direction) } if (conn) { - // bpf_dbg_printk("conn pid %d", conn.pid); - // dbg_print_http_connection_info(&conn->p_conn.conn); + bpf_dbg_printk("SSL conn"); + dbg_print_http_connection_info(&conn->p_conn.conn); // unsigned char buf[48]; // bpf_probe_read(buf, 48, (void *)args->buf); // for (int i=0; i < 48; i++) { // bpf_dbg_printk("%x ", buf[i]); // } - bpf_map_update_elem(&active_ssl_connections, &conn->p_conn, &ssl_ptr, BPF_ANY); // We should attempt to clean up the server trace immediately. The cleanup information // is keyed of the *ssl, so when it's delayed we might have different *ssl on the same @@ -215,7 +214,11 @@ handle_ssl_buf(void *ctx, u64 id, ssl_args_t *args, int bytes_len, u8 direction) } } -static __always_inline void *is_ssl_connection(u64 id) { +static __always_inline void set_active_ssl_connection(pid_connection_info_t *conn, void *ssl) { + bpf_map_update_elem(&active_ssl_connections, conn, &ssl, BPF_ANY); +} + +static __always_inline void *is_ssl_connection(u64 id, pid_connection_info_t *conn) { void *ssl = 0; // Checks if it's sandwitched between active SSL handshake, read or write uprobe/uretprobe void **s = bpf_map_lookup_elem(&active_ssl_handshakes, &id); @@ -231,11 +234,13 @@ static __always_inline void *is_ssl_connection(u64 id) { } } - return ssl; -} + if (!ssl) { + return bpf_map_lookup_elem(&active_ssl_connections, conn); + } else { + set_active_ssl_connection(conn, ssl); + } -static __always_inline void *is_active_ssl(pid_connection_info_t *conn) { - return bpf_map_lookup_elem(&active_ssl_connections, conn); + return ssl; } #endif diff --git a/bpf/k_tracer.h b/bpf/k_tracer.h index eab76860b..29804a649 100644 --- a/bpf/k_tracer.h +++ b/bpf/k_tracer.h @@ -282,55 +282,50 @@ int BPF_KPROBE(kprobe_tcp_sendmsg, struct sock *sk, struct msghdr *msg, size_t s sort_connection_info(&s_args.p_conn.conn); s_args.p_conn.pid = pid_from_pid_tgid(id); - void *ssl = is_ssl_connection(id); + void *ssl = is_ssl_connection(id, &s_args.p_conn); if (size > 0) { if (!ssl) { - void *active_ssl = is_active_ssl(&s_args.p_conn); - if (!active_ssl) { - u8 *buf = iovec_memory(); - if (buf) { - size = read_msghdr_buf(msg, buf, size); - // If a sock_msg program is installed, this kprobe will fail to - // read anything, because the data is in bvec physical pages. However, - // the sock_msg will setup a buffer for us if this is the case. We - // look up this buffer and use it instead of what we'd get from - // calling read_msghdr_buf. - if (!size) { - msg_buffer_t *m_buf = bpf_map_lookup_elem(&msg_buffers, &e_key); - bpf_dbg_printk("No size, m_buf[%llx]", m_buf); - if (m_buf) { - buf = m_buf->buf; - // The buffer setup for us by a sock_msg program is always the - // full buffer, but when we extend a packet to be able to inject - // a Traceparent field, it will actually be split in 3 chunks: - // [before the injected header],[70 bytes for 'Traceparent...'],[the rest]. - // We don't want the handle_buf_with_connection logic to run more than - // once on the same data, so if we find a buf we send all of it to the - // handle_buf_with_connection logic and then mark it as seen by making - // m_buf->pos be the size of the buffer. - if (!m_buf->pos) { - size = sizeof(m_buf->buf); - m_buf->pos = size; - bpf_dbg_printk("msg_buffer: size %d, buf[%s]", size, buf); - } else { - size = 0; - } + u8 *buf = iovec_memory(); + if (buf) { + size = read_msghdr_buf(msg, buf, size); + // If a sock_msg program is installed, this kprobe will fail to + // read anything, because the data is in bvec physical pages. However, + // the sock_msg will setup a buffer for us if this is the case. We + // look up this buffer and use it instead of what we'd get from + // calling read_msghdr_buf. + if (!size) { + msg_buffer_t *m_buf = bpf_map_lookup_elem(&msg_buffers, &e_key); + bpf_dbg_printk("No size, m_buf[%llx]", m_buf); + if (m_buf) { + buf = m_buf->buf; + // The buffer setup for us by a sock_msg program is always the + // full buffer, but when we extend a packet to be able to inject + // a Traceparent field, it will actually be split in 3 chunks: + // [before the injected header],[70 bytes for 'Traceparent...'],[the rest]. + // We don't want the handle_buf_with_connection logic to run more than + // once on the same data, so if we find a buf we send all of it to the + // handle_buf_with_connection logic and then mark it as seen by making + // m_buf->pos be the size of the buffer. + if (!m_buf->pos) { + size = sizeof(m_buf->buf); + m_buf->pos = size; + bpf_dbg_printk("msg_buffer: size %d, buf[%s]", size, buf); + } else { + size = 0; } } - if (size) { - u64 sock_p = (u64)sk; - bpf_map_update_elem(&active_send_args, &id, &s_args, BPF_ANY); - bpf_map_update_elem(&active_send_sock_args, &sock_p, &s_args, BPF_ANY); - - // Logically last for !ssl. - handle_buf_with_connection( - ctx, &s_args.p_conn, buf, size, NO_SSL, TCP_SEND, orig_dport); - } else { - bpf_dbg_printk("can't find iovec ptr in msghdr, not tracking sendmsg"); - } } - } else { - bpf_dbg_printk("tcp_sendmsg for identified SSL connection, ignoring..."); + if (size) { + u64 sock_p = (u64)sk; + bpf_map_update_elem(&active_send_args, &id, &s_args, BPF_ANY); + bpf_map_update_elem(&active_send_sock_args, &sock_p, &s_args, BPF_ANY); + + // Logically last for !ssl. + handle_buf_with_connection( + ctx, &s_args.p_conn, buf, size, NO_SSL, TCP_SEND, orig_dport); + } else { + bpf_dbg_printk("can't find iovec ptr in msghdr, not tracking sendmsg"); + } } } else { bpf_dbg_printk("tcp_sendmsg for identified SSL connection, ignoring..."); @@ -487,24 +482,19 @@ static __always_inline int return_recvmsg(void *ctx, u64 id, int copied_len) { sort_connection_info(&info.conn); info.pid = pid_from_pid_tgid(id); - void *ssl = is_ssl_connection(id); + void *ssl = is_ssl_connection(id, &info); if (!ssl) { - void *active_ssl = is_active_ssl(&info); - if (!active_ssl) { - u8 *buf = iovec_memory(); - if (buf) { - copied_len = read_iovec_ctx(iov_ctx, buf, copied_len); - if (copied_len) { - // doesn't return must be logically last statement - handle_buf_with_connection( - ctx, &info, buf, copied_len, NO_SSL, TCP_RECV, orig_dport); - } else { - bpf_dbg_printk("Not copied anything"); - } + u8 *buf = iovec_memory(); + if (buf) { + copied_len = read_iovec_ctx(iov_ctx, buf, copied_len); + if (copied_len) { + // doesn't return must be logically last statement + handle_buf_with_connection( + ctx, &info, buf, copied_len, NO_SSL, TCP_RECV, orig_dport); + } else { + bpf_dbg_printk("Not copied anything"); } - } else { - bpf_dbg_printk("tcp_recvmsg for an identified SSL connection, ignoring..."); } } else { bpf_dbg_printk("tcp_recvmsg for an identified SSL connection, ignoring..."); @@ -680,7 +670,6 @@ int BPF_KPROBE(kprobe_sys_exit, int status) { if (s_args) { bpf_dbg_printk("Checking if we need to finish the request per thread id"); finish_possible_delayed_http_request(&s_args->p_conn); - bpf_map_delete_elem(&active_ssl_connections, &s_args->p_conn); } bpf_map_delete_elem(&clone_map, &task.p_key); diff --git a/bpf/protocol_tcp.h b/bpf/protocol_tcp.h index 6943ec067..2fda800a0 100644 --- a/bpf/protocol_tcp.h +++ b/bpf/protocol_tcp.h @@ -91,6 +91,12 @@ static __always_inline void handle_unknown_tcp_connection(pid_connection_info_t u8 ssl, u16 orig_dport) { tcp_req_t *existing = bpf_map_lookup_elem(&ongoing_tcp_req, pid_conn); + if (existing) { + if (existing->direction == direction && existing->end_monotime_ns != 0) { + bpf_map_delete_elem(&ongoing_tcp_req, pid_conn); + existing = 0; + } + } if (!existing) { tcp_req_t *req = empty_tcp_req(); if (req) { @@ -100,6 +106,8 @@ static __always_inline void handle_unknown_tcp_connection(pid_connection_info_t req->ssl = ssl; req->direction = direction; req->start_monotime_ns = bpf_ktime_get_ns(); + req->end_monotime_ns = 0; + req->resp_len = 0; req->len = bytes_len; task_pid(&req->pid); bpf_probe_read(req->buf, K_TCP_MAX_LEN, u_buf); @@ -113,18 +121,19 @@ static __always_inline void handle_unknown_tcp_connection(pid_connection_info_t bpf_map_update_elem(&ongoing_tcp_req, pid_conn, req, BPF_ANY); } } else if (existing->direction != direction) { - existing->end_monotime_ns = bpf_ktime_get_ns(); - existing->resp_len = bytes_len; - tcp_req_t *trace = bpf_ringbuf_reserve(&events, sizeof(tcp_req_t), 0); - if (trace) { - bpf_dbg_printk( - "Sending TCP trace %lx, response length %d", existing, existing->resp_len); - - __builtin_memcpy(trace, existing, sizeof(tcp_req_t)); - bpf_probe_read(trace->rbuf, K_TCP_RES_LEN, u_buf); - bpf_ringbuf_submit(trace, get_flags()); + if (existing->end_monotime_ns == 0) { + existing->end_monotime_ns = bpf_ktime_get_ns(); + existing->resp_len = bytes_len; + tcp_req_t *trace = bpf_ringbuf_reserve(&events, sizeof(tcp_req_t), 0); + if (trace) { + bpf_dbg_printk( + "Sending TCP trace %lx, response length %d", existing, existing->resp_len); + + __builtin_memcpy(trace, existing, sizeof(tcp_req_t)); + bpf_probe_read(trace->rbuf, K_TCP_RES_LEN, u_buf); + bpf_ringbuf_submit(trace, get_flags()); + } } - bpf_map_delete_elem(&ongoing_tcp_req, pid_conn); } else if (existing->len > 0 && existing->len < (K_TCP_MAX_LEN / 2)) { // Attempt to append one more packet. I couldn't convince the verifier // to use a variable (K_TCP_MAX_LEN-existing->len). If needed we may need diff --git a/bpf/tc_sock.h b/bpf/tc_sock.h index b90a9050f..e8ffd54c3 100644 --- a/bpf/tc_sock.h +++ b/bpf/tc_sock.h @@ -193,26 +193,23 @@ static __always_inline u8 protocol_detector(struct sk_msg_md *msg, sort_connection_info(&s_args.p_conn.conn); s_args.p_conn.pid = pid_from_pid_tgid(id); - void *ssl = is_ssl_connection(id); + void *ssl = is_ssl_connection(id, &s_args.p_conn); if (s_args.size > 0) { if (!ssl) { - void *active_ssl = is_active_ssl(&s_args.p_conn); - if (!active_ssl) { - msg_buffer_t msg_buf = { - .pos = 0, - }; - bpf_probe_read_kernel(msg_buf.buf, FULL_BUF_SIZE, msg->data); - // We setup any call that looks like HTTP request to be extended. - // This must match exactly to what the decision will be for - // the kprobe program on tcp_sendmsg, which sets up the - // outgoing_trace_map data used by Traffic Control to write the - // actual 'Traceparent:...' string. - if (is_http_request_buf((const unsigned char *)msg_buf.buf)) { - bpf_dbg_printk("Setting up request to be extended"); - bpf_map_update_elem(&msg_buffers, &e_key, &msg_buf, BPF_ANY); - - return 1; - } + msg_buffer_t msg_buf = { + .pos = 0, + }; + bpf_probe_read_kernel(msg_buf.buf, FULL_BUF_SIZE, msg->data); + // We setup any call that looks like HTTP request to be extended. + // This must match exactly to what the decision will be for + // the kprobe program on tcp_sendmsg, which sets up the + // outgoing_trace_map data used by Traffic Control to write the + // actual 'Traceparent:...' string. + if (is_http_request_buf((const unsigned char *)msg_buf.buf)) { + bpf_dbg_printk("Setting up request to be extended"); + bpf_map_update_elem(&msg_buffers, &e_key, &msg_buf, BPF_ANY); + + return 1; } } } diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index 64c422df8..aabafe360 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -504,7 +504,8 @@ generation of Beyla metrics. Configures the time interval after which an HTTP request is considered as a timeout. This option allows Beyla to report HTTP transactions which timeout and never return. -To disable the automatic HTTP request timeout feature, set this option to zero, i.e. "0ms". +To disable the automatic HTTP request timeout feature, set this option to zero, +that is "0ms". | YAML | Environment variable | Type | Default | | ----------------------- | ---------------------------------- | -------- | ------- | @@ -514,6 +515,19 @@ Configures the HTTP tracer heuristic to send telemetry events as soon as a respo Setting this option reduces the accuracy of timings for requests with large responses, however, in high request volume scenarios this option will reduce the number of dropped trace events. +| YAML | Environment variable | Type | Default | +| ----------------------- | ---------------------------------- | -------- | ------- | +| `heuristic_sql_detect` | `BEYLA_HEURISTIC_SQL_DETECT` | boolean | (false) | + +By default, Beyla detects various SQL client requests through detection of their +particular binary protocol format. However, oftentimes SQL database clients send their +queries in a format where Beyla can detect the query statement without knowing +the exact binary protocol. If you are using a database technology not directly supported +by Beyla, you can enable this option to get database client telemetry. The option is +not enabled by default, because it can create false positives, for example, an application +sending SQL text for logging purposes through a TCP connection. Currently supported +protocols where this option isn't needed are the Postgres and MySQL binary protocols. + ## Configuration of metrics and traces attributes Grafana Beyla allows configuring how some attributes for metrics and traces diff --git a/pkg/config/ebpf_tracer.go b/pkg/config/ebpf_tracer.go index 61b96da33..cc2b4217f 100644 --- a/pkg/config/ebpf_tracer.go +++ b/pkg/config/ebpf_tracer.go @@ -36,4 +36,8 @@ type EPPFTracer struct { // Optimises for getting requests information immediately when request response is seen HighRequestVolume bool `yaml:"high_request_volume" env:"BEYLA_BPF_HIGH_REQUEST_VOLUME"` + + // Enables the heuristic based detection of SQL requests. This can be used to detect + // talking to databases other than the ones we recognize in Beyla, like Postgres and MySQL + HeuristicSQLDetect bool `yaml:"heuristic_sql_detect" env:"BEYLA_HEURISTIC_SQL_DETECT"` } diff --git a/pkg/export/otel/traces.go b/pkg/export/otel/traces.go index 9caed4af6..d05c95ab3 100644 --- a/pkg/export/otel/traces.go +++ b/pkg/export/otel/traces.go @@ -626,7 +626,7 @@ func traceAttributes(span *request.Span, optionalAttrs map[attr.Name]struct{}) [ attrs = []attribute.KeyValue{ request.ServerAddr(request.HostAsServer(span)), request.ServerPort(span.HostPort), - semconv.DBSystemOtherSQL, // We can distinguish in the future for MySQL, Postgres etc + span.DBSystem(), // We can distinguish in the future for MySQL, Postgres etc } if _, ok := optionalAttrs[attr.DBQueryText]; ok { attrs = append(attrs, request.DBQueryText(span.Statement)) diff --git a/pkg/internal/ebpf/common/common.go b/pkg/internal/ebpf/common/common.go index 29fa15b03..c225e4656 100644 --- a/pkg/internal/ebpf/common/common.go +++ b/pkg/internal/ebpf/common/common.go @@ -14,6 +14,7 @@ import ( "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/ringbuf" + "github.com/grafana/beyla/pkg/config" "github.com/grafana/beyla/pkg/internal/goexec" "github.com/grafana/beyla/pkg/internal/request" ) @@ -86,7 +87,7 @@ var MisclassifiedEvents = make(chan MisclassifiedEvent) func ptlog() *slog.Logger { return slog.With("component", "ebpf.ProcessTracer") } -func ReadBPFTraceAsSpan(record *ringbuf.Record, filter ServiceFilter) (request.Span, bool, error) { +func ReadBPFTraceAsSpan(cfg *config.EPPFTracer, record *ringbuf.Record, filter ServiceFilter) (request.Span, bool, error) { var eventType uint8 // we read the type first, depending on the type we decide what kind of record we have @@ -103,7 +104,7 @@ func ReadBPFTraceAsSpan(record *ringbuf.Record, filter ServiceFilter) (request.S case EventTypeKHTTP2: return ReadHTTP2InfoIntoSpan(record, filter) case EventTypeTCP: - return ReadTCPRequestIntoSpan(record, filter) + return ReadTCPRequestIntoSpan(cfg, record, filter) case EventTypeGoSarama: return ReadGoSaramaRequestIntoSpan(record) case EventTypeGoRedis: diff --git a/pkg/internal/ebpf/common/ringbuf.go b/pkg/internal/ebpf/common/ringbuf.go index 0ffd6a9b1..3a33e4c5e 100644 --- a/pkg/internal/ebpf/common/ringbuf.go +++ b/pkg/internal/ebpf/common/ringbuf.go @@ -38,7 +38,7 @@ type ringBufForwarder struct { spansLen int access sync.Mutex ticker *time.Ticker - reader func(*ringbuf.Record, ServiceFilter) (request.Span, bool, error) + reader func(*config.EPPFTracer, *ringbuf.Record, ServiceFilter) (request.Span, bool, error) // filter the input spans, eliminating these from processes whose PID // belong to a process that does not match the discovery policies filter ServiceFilter @@ -78,7 +78,7 @@ func ForwardRingbuf( cfg *config.EPPFTracer, ringbuffer *ebpf.Map, filter ServiceFilter, - reader func(*ringbuf.Record, ServiceFilter) (request.Span, bool, error), + reader func(*config.EPPFTracer, *ringbuf.Record, ServiceFilter) (request.Span, bool, error), logger *slog.Logger, metrics imetrics.Reporter, closers ...io.Closer, @@ -171,7 +171,7 @@ func (rbf *ringBufForwarder) alreadyForwarded(ctx context.Context, _ []io.Closer func (rbf *ringBufForwarder) processAndForward(record ringbuf.Record, spansChan chan<- []request.Span) { rbf.access.Lock() defer rbf.access.Unlock() - s, ignore, err := rbf.reader(&record, rbf.filter) + s, ignore, err := rbf.reader(rbf.cfg, &record, rbf.filter) if err != nil { rbf.logger.Error("error parsing perf event", "error", err) return diff --git a/pkg/internal/ebpf/common/sql_detect_mysql.go b/pkg/internal/ebpf/common/sql_detect_mysql.go new file mode 100644 index 000000000..e00b069e0 --- /dev/null +++ b/pkg/internal/ebpf/common/sql_detect_mysql.go @@ -0,0 +1,77 @@ +package ebpfcommon + +import ( + "encoding/binary" + "strings" +) + +type mySQLHdr struct { + length uint32 // payload length + sequence ID + command uint8 // command type +} + +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query.html +const kMySQLQuery = uint8(0x3) + +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_prepare.html +const kMySQLPrepare = uint8(0x16) + +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html +const kMySQLExecute = uint8(0x17) + +func isMySQL(b []byte) bool { + return isValidMySQLPayload(b) +} + +func readMySQLHeader(b []byte) (mySQLHdr, error) { + hdr := mySQLHdr{} + + hdr.length = binary.LittleEndian.Uint32(b[:4]) + hdr.length &= 0x00ffffff // remove the sequence id from the length + hdr.command = uint8(b[4]) + + return hdr, nil +} + +func isValidMySQLPayload(b []byte) bool { + // the header is at least 5 bytes + if len(b) < 6 { + return false + } + + hdr, err := readMySQLHeader(b) + if err != nil { + return false + } + + if hdr.length == 0 { + return false + } + + return hdr.command == kMySQLQuery || hdr.command == kMySQLPrepare || hdr.command == kMySQLExecute +} + +func mysqlPreparedStatements(b []byte) (string, string, string) { + text := string(b) + query := asciiToUpper(text) + execIdx := strings.Index(query, "EXECUTE ") + if execIdx < 0 { + return "", "", "" + } + + if execIdx >= len(text) { + return "", "", "" + } + + text = text[execIdx:] + + parts := strings.Split(text, " ") + op := parts[0] + var table string + if len(parts) > 1 { + table = parts[1] + } + sql := text + + return op, table, sql +} diff --git a/pkg/internal/ebpf/common/sql_detect_mysql_test.go b/pkg/internal/ebpf/common/sql_detect_mysql_test.go new file mode 100644 index 000000000..0905ebc9f --- /dev/null +++ b/pkg/internal/ebpf/common/sql_detect_mysql_test.go @@ -0,0 +1,80 @@ +package ebpfcommon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMySQLParsing(t *testing.T) { + for _, ts := range []struct { + name string + bytes []byte + valid bool + hasError bool + result mySQLHdr + }{ + { + name: "Valid prepare", + bytes: []byte{0x1c, 0x00, 0x00, 0x00, 0x16, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x43, 0x4f, 0x4e, 0x43, 0x41, 0x54, 0x28, 0x3f, 0x2c, 0x20, 0x3f, 0x29, 0x20, 0x41, 0x53, 0x20, 0x63, 0x6f, 0x6c, 0x31}, + valid: true, + hasError: false, + result: mySQLHdr{ + length: 0x1c, + command: 0x16, + }, + }, + { + name: "Valid execute", + bytes: []byte{0x12, 0x00, 0x00, 0x00, 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x0f, 0x00, 0x03, 0x66, 0x6f, 0x6f}, + valid: true, + hasError: false, + result: mySQLHdr{ + length: 0x12, + command: 0x17, + }, + }, + { + name: "Valid Query", + bytes: []byte{0x21, 0x00, 0x00, 0x01, 0x03, 0x01, 0x01, 0x00, 0x01, 0xfe, 0x00, 0x01, 0x61, 0x01, 0x31, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x20, 0x40, 0x40, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x20, 0x31}, + valid: true, + hasError: false, + result: mySQLHdr{ + length: 0x21, + command: 0x3, + }, + }, + { + name: "Unknown opcode", + bytes: []byte{0x1c, 0x00, 0x00, 0x00, 0x10, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x43, 0x4f, 0x4e, 0x43, 0x41, 0x54, 0x28, 0x3f, 0x2c, 0x20, 0x3f, 0x29, 0x20, 0x41, 0x53, 0x20, 0x63, 0x6f, 0x6c, 0x31}, + valid: false, + hasError: false, + result: mySQLHdr{ + length: 0x1c, + command: 0x10, + }, + }, + { + name: "Zero size", + bytes: []byte{0x00, 0x00, 0x00, 0x00, 0x16, 0x53, 0x45, 0x4c, 0x45, 0x43, 0x54, 0x20, 0x43, 0x4f, 0x4e, 0x43, 0x41, 0x54, 0x28, 0x3f, 0x2c, 0x20, 0x3f, 0x29, 0x20, 0x41, 0x53, 0x20, 0x63, 0x6f, 0x6c, 0x31}, + valid: false, + hasError: false, + result: mySQLHdr{ + length: 0, + command: 0x16, + }, + }, + } { + t.Run(ts.name, func(t *testing.T) { + hdr, err := readMySQLHeader(ts.bytes) + if ts.hasError { + assert.NotNil(t, err) + return + } else { + assert.Nil(t, err) + } + assert.Equal(t, ts.result, hdr) + assert.Equal(t, ts.valid, isMySQL(ts.bytes)) + }) + } +} diff --git a/pkg/internal/ebpf/common/sql_detect_postgres.go b/pkg/internal/ebpf/common/sql_detect_postgres.go new file mode 100644 index 000000000..e72ba489a --- /dev/null +++ b/pkg/internal/ebpf/common/sql_detect_postgres.go @@ -0,0 +1,165 @@ +package ebpfcommon + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" +) + +const kPostgresBind = byte('B') +const kPostgresQuery = byte('Q') +const kPostgresCommand = byte('C') + +func isPostgres(b []byte) bool { + op, ok := isValidPostgresPayload(b) + + return ok && (op == kPostgresQuery || op == kPostgresCommand || op == kPostgresBind) +} + +func isPostgresBindCommand(b []byte) bool { + op, ok := isValidPostgresPayload(b) + + return ok && (op == kPostgresBind) +} + +func isPostgresQueryCommand(b []byte) bool { + op, ok := isValidPostgresPayload(b) + + return ok && (op == kPostgresQuery) +} + +func isValidPostgresPayload(b []byte) (byte, bool) { + // https://github.com/postgres/postgres/blob/master/src/interfaces/libpq/fe-protocol3.c#L97 + if len(b) < 5 { + return 0, false + } + + size := int32(binary.BigEndian.Uint32(b[1:5])) + if size < 0 || size > 3000 { + return 0, false + } + + return b[0], true +} + +// nolint:cyclop +func parsePostgresBindCommand(buf []byte) (string, string, []string, error) { + statement := []byte{} + portal := []byte{} + args := []string{} + + size := int(binary.BigEndian.Uint32(buf[1:5])) + if size > len(buf) { + size = len(buf) + } + ptr := 5 + + // parse statement, zero terminated string + for { + if ptr >= size { + return string(statement), string(portal), args, errors.New("too short, while parsing statement") + } + b := buf[ptr] + ptr++ + + if b == 0 { + break + } + statement = append(statement, b) + } + + // parse portal, zero terminated string + for { + if ptr >= size { + return string(statement), string(portal), args, errors.New("too short, while parsing portal") + } + b := buf[ptr] + ptr++ + + if b == 0 { + break + } + portal = append(portal, b) + } + + if ptr+2 >= size { + return string(statement), string(portal), args, errors.New("too short, while parsing format codes") + } + + formats := int16(binary.BigEndian.Uint16(buf[ptr : ptr+2])) + ptr += 2 + for i := 0; i < int(formats); i++ { + // ignore format codes + if ptr+2 >= size { + return string(statement), string(portal), args, errors.New("too short, while parsing format codes") + } + ptr += 2 + } + + params := int16(binary.BigEndian.Uint16(buf[ptr : ptr+2])) + ptr += 2 + for i := 0; i < int(params); i++ { + if ptr+4 >= size { + return string(statement), string(portal), args, errors.New("too short, while parsing params") + } + argLen := int(binary.BigEndian.Uint32(buf[ptr : ptr+4])) + ptr += 4 + arg := []byte{} + for j := 0; j < int(argLen); j++ { + if ptr >= size { + break + } + arg = append(arg, buf[ptr]) + ptr++ + } + args = append(args, string(arg)) + } + + return string(statement), string(portal), args, nil +} + +func parsePosgresQueryCommand(buf []byte) (string, error) { + size := int(binary.BigEndian.Uint32(buf[1:5])) + if size > len(buf) { + size = len(buf) + } + ptr := 5 + + if ptr > size { + return "", errors.New("too short") + } + + return string(buf[ptr:size]), nil +} + +func postgresPreparedStatements(b []byte) (string, string, string) { + var op, table, sql string + if isPostgresBindCommand(b) { + statement, portal, args, err := parsePostgresBindCommand(b) + if err == nil { + op = "PREPARED STATEMENT" + table = fmt.Sprintf("%s.%s", statement, portal) + for _, arg := range args { + if isASCII(arg) { + sql += arg + " " + } + } + } + } else if isPostgresQueryCommand(b) { + text, err := parsePosgresQueryCommand(b) + if err == nil { + query := asciiToUpper(text) + if strings.HasPrefix(query, "EXECUTE ") { + parts := strings.Split(text, " ") + op = parts[0] + if len(parts) > 1 { + table = parts[1] + } + sql = text + } + } + } + + return op, table, sql +} diff --git a/pkg/internal/ebpf/common/sql_detect_transform.go b/pkg/internal/ebpf/common/sql_detect_transform.go index c01d670b8..00921e024 100644 --- a/pkg/internal/ebpf/common/sql_detect_transform.go +++ b/pkg/internal/ebpf/common/sql_detect_transform.go @@ -1,9 +1,6 @@ package ebpfcommon import ( - "encoding/binary" - "errors" - "fmt" "strings" "unsafe" @@ -13,6 +10,16 @@ import ( "github.com/grafana/beyla/pkg/internal/sqlprune" ) +func sqlKind(b []byte) request.SQLKind { + if isPostgres(b) { + return request.DBPostgres + } else if isMySQL(b) { + return request.DBMySQL + } + + return request.DBGeneric +} + func validSQL(op, table string) bool { return op != "" && table != "" } @@ -45,37 +52,24 @@ func isASCII(s string) bool { return true } -func detectSQLBytes(b []byte) (string, string, string) { +func detectSQLPayload(useHeuristics bool, b []byte) (string, string, string, request.SQLKind) { + sqlKind := sqlKind(b) + if !useHeuristics { + if sqlKind == request.DBGeneric { + return "", "", "", sqlKind + } + } op, table, sql := detectSQL(string(b)) if !validSQL(op, table) { - if isPostgresBindCommand(b) { - statement, portal, args, err := parsePostgresBindCommand(b) - if err == nil { - op = "BIND" - table = fmt.Sprintf("%s.%s", statement, portal) - for _, arg := range args { - if isASCII(arg) { - sql += arg + " " - } - } - } - } else if isPostgresQueryCommand(b) { - text, err := parsePosgresQueryCommand(b) - if err == nil { - query := asciiToUpper(text) - if strings.HasPrefix(query, "EXECUTE ") { - parts := strings.Split(text, " ") - op = parts[0] - if len(parts) > 1 { - table = parts[1] - } - sql = text - } - } + switch sqlKind { + case request.DBPostgres: + op, table, sql = postgresPreparedStatements(b) + case request.DBMySQL: + op, table, sql = mysqlPreparedStatements(b) } } - return op, table, sql + return op, table, sql, sqlKind } func detectSQL(buf string) (string, string, string) { @@ -93,121 +87,7 @@ func detectSQL(buf string) (string, string, string) { return "", "", "" } -func isPostgresBindCommand(b []byte) bool { - return isPostgresCommand('B', b) -} - -func isPostgresQueryCommand(b []byte) bool { - return isPostgresCommand('Q', b) -} - -func isPostgresCommand(lookup byte, b []byte) bool { - if len(b) < 5 { - return false - } - - if b[0] == lookup { - size := int32(binary.BigEndian.Uint32(b[1:5])) - if size < 0 || size > 1000 { - return false - } - return true - } - - return false -} - -// nolint:cyclop -func parsePostgresBindCommand(buf []byte) (string, string, []string, error) { - statement := []byte{} - portal := []byte{} - args := []string{} - - size := int(binary.BigEndian.Uint32(buf[1:5])) - if size > len(buf) { - size = len(buf) - } - ptr := 5 - - // parse statement, zero terminated string - for { - if ptr >= size { - return string(statement), string(portal), args, errors.New("too short, while parsing statement") - } - b := buf[ptr] - ptr++ - - if b == 0 { - break - } - statement = append(statement, b) - } - - // parse portal, zero terminated string - for { - if ptr >= size { - return string(statement), string(portal), args, errors.New("too short, while parsing portal") - } - b := buf[ptr] - ptr++ - - if b == 0 { - break - } - portal = append(portal, b) - } - - if ptr+2 >= size { - return string(statement), string(portal), args, errors.New("too short, while parsing format codes") - } - - formats := int16(binary.BigEndian.Uint16(buf[ptr : ptr+2])) - ptr += 2 - for i := 0; i < int(formats); i++ { - // ignore format codes - if ptr+2 >= size { - return string(statement), string(portal), args, errors.New("too short, while parsing format codes") - } - ptr += 2 - } - - params := int16(binary.BigEndian.Uint16(buf[ptr : ptr+2])) - ptr += 2 - for i := 0; i < int(params); i++ { - if ptr+4 >= size { - return string(statement), string(portal), args, errors.New("too short, while parsing params") - } - argLen := int(binary.BigEndian.Uint32(buf[ptr : ptr+4])) - ptr += 4 - arg := []byte{} - for j := 0; j < int(argLen); j++ { - if ptr >= size { - break - } - arg = append(arg, buf[ptr]) - ptr++ - } - args = append(args, string(arg)) - } - - return string(statement), string(portal), args, nil -} - -func parsePosgresQueryCommand(buf []byte) (string, error) { - size := int(binary.BigEndian.Uint32(buf[1:5])) - if size > len(buf) { - size = len(buf) - } - ptr := 5 - - if ptr > size { - return "", errors.New("too short") - } - - return string(buf[ptr:size]), nil -} - -func TCPToSQLToSpan(trace *TCPRequestInfo, op, table, sql string) request.Span { +func TCPToSQLToSpan(trace *TCPRequestInfo, op, table, sql string, kind request.SQLKind) request.Span { peer := "" peerPort := 0 hostname := "" @@ -227,7 +107,7 @@ func TCPToSQLToSpan(trace *TCPRequestInfo, op, table, sql string) request.Span { PeerPort: peerPort, Host: hostname, HostPort: hostPort, - ContentLength: 0, + ContentLength: int64(trace.Len), RequestStart: int64(trace.StartMonotimeNs), Start: int64(trace.StartMonotimeNs), End: int64(trace.EndMonotimeNs), @@ -242,5 +122,6 @@ func TCPToSQLToSpan(trace *TCPRequestInfo, op, table, sql string) request.Span { Namespace: trace.Pid.Ns, }, Statement: sql, + SubType: int(kind), } } diff --git a/pkg/internal/ebpf/common/sql_detect_transform_test.go b/pkg/internal/ebpf/common/sql_detect_transform_test.go index 10995058c..af065f0be 100644 --- a/pkg/internal/ebpf/common/sql_detect_transform_test.go +++ b/pkg/internal/ebpf/common/sql_detect_transform_test.go @@ -158,9 +158,21 @@ func TestPostgresQueryParsing(t *testing.T) { table: "", sql: "", }, + { + name: "MySQL prepared statement", + bytes: []byte{36, 0, 0, 0, 3, 0, 1, 69, 88, 69, 67, 85, 84, 69, 32, 109, 121, 95, 97, 99, 116, 111, 114, 115, 32, 85, 83, 73, 78, 71, 32, 64, 97, 99, 116, 111, 114, 95, 105, 100}, + op: "EXECUTE", + table: "my_actors", + sql: "EXECUTE my_actors USING @actor_id", + }, } { t.Run(ts.name, func(t *testing.T) { - op, table, sql := detectSQLBytes(ts.bytes) + op, table, sql, _ := detectSQLPayload(false, ts.bytes) + assert.Equal(t, ts.op, op) + assert.Equal(t, ts.table, table) + assert.Equal(t, ts.sql, sql) + + op, table, sql, _ = detectSQLPayload(true, ts.bytes) assert.Equal(t, ts.op, op) assert.Equal(t, ts.table, table) assert.Equal(t, ts.sql, sql) diff --git a/pkg/internal/ebpf/common/tcp_detect_transform.go b/pkg/internal/ebpf/common/tcp_detect_transform.go index efada79e8..dd332e2d9 100644 --- a/pkg/internal/ebpf/common/tcp_detect_transform.go +++ b/pkg/internal/ebpf/common/tcp_detect_transform.go @@ -6,11 +6,12 @@ import ( "github.com/cilium/ebpf/ringbuf" + "github.com/grafana/beyla/pkg/config" "github.com/grafana/beyla/pkg/internal/request" ) // nolint:cyclop -func ReadTCPRequestIntoSpan(record *ringbuf.Record, filter ServiceFilter) (request.Span, bool, error) { +func ReadTCPRequestIntoSpan(cfg *config.EPPFTracer, record *ringbuf.Record, filter ServiceFilter) (request.Span, bool, error) { var event TCPRequestInfo err := binary.Read(bytes.NewBuffer(record.RawSample), binary.LittleEndian, &event) @@ -35,9 +36,14 @@ func ReadTCPRequestIntoSpan(record *ringbuf.Record, filter ServiceFilter) (reque b := event.Buf[:l] // Check if we have a SQL statement - op, table, sql := detectSQLBytes(b) + op, table, sql, kind := detectSQLPayload(cfg.HeuristicSQLDetect, b) if validSQL(op, table) { - return TCPToSQLToSpan(&event, op, table, sql), false, nil + return TCPToSQLToSpan(&event, op, table, sql, kind), false, nil + } else { + op, table, sql, kind = detectSQLPayload(cfg.HeuristicSQLDetect, event.Rbuf[:rl]) + if validSQL(op, table) { + return TCPToSQLToSpan(&event, op, table, sql, kind), false, nil + } } if maybeFastCGI(b) { diff --git a/pkg/internal/ebpf/common/tcp_detect_transform_test.go b/pkg/internal/ebpf/common/tcp_detect_transform_test.go index 3179e7bc2..1ac49e010 100644 --- a/pkg/internal/ebpf/common/tcp_detect_transform_test.go +++ b/pkg/internal/ebpf/common/tcp_detect_transform_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/grafana/beyla/pkg/config" "github.com/grafana/beyla/pkg/internal/request" "github.com/grafana/beyla/pkg/internal/svc" ) @@ -27,7 +28,7 @@ func TestTCPReqSQLParsing(t *testing.T) { op, table, sql := detectSQL(sql) assert.Equal(t, op, "SELECT") assert.Equal(t, table, "accounts") - s := TCPToSQLToSpan(&r, op, table, sql) + s := TCPToSQLToSpan(&r, op, table, sql, request.DBGeneric) assert.NotNil(t, s) assert.NotEmpty(t, s.Host) assert.NotEmpty(t, s.Peer) @@ -97,9 +98,12 @@ func TestReadTCPRequestIntoSpan_Overflow(t *testing.T) { 169, 193, 172, 206, 225, 219, 112, 52, 115, 32, 147, 192, 127, 211, 129, 241, }, } + + cfg := config.EPPFTracer{HeuristicSQLDetect: true} + binaryRecord := bytes.Buffer{} require.NoError(t, binary.Write(&binaryRecord, binary.LittleEndian, tri)) - span, ignore, err := ReadTCPRequestIntoSpan(&ringbuf.Record{RawSample: binaryRecord.Bytes()}, &fltr) + span, ignore, err := ReadTCPRequestIntoSpan(&cfg, &ringbuf.Record{RawSample: binaryRecord.Bytes()}, &fltr) require.NoError(t, err) require.False(t, ignore) diff --git a/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o index 53fc43261..881d8d343 100644 --- a/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2abff0827a0fa68c39db2dab7e06f526320eed0b830e8d12c711291b1f3504a1 -size 535592 +oid sha256:3e1cf63db03527c347efbf8915f52d78626ddca3f903a64883196a04a6cf4b5c +size 536488 diff --git a/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o index 0261cedd1..c9d5245ae 100644 --- a/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ba13d5b910fe88ef0583ae978d0e80b0325014c8dcb61cfb9578c84fb9589d00 -size 898240 +oid sha256:1ed41d825128c3175044f335e9f20a75794dd013c6bc6cdb9ae25a3272cb0bd8 +size 915472 diff --git a/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o index d55397614..97a7efeab 100644 --- a/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d624cf08b2b1a73cb5c4807a6b9f72c23650b0d2eca98ab19da674718014f6bf -size 897752 +oid sha256:61e5375aa892a9f8c6554290b23e355f30a0b1beb6ae74147b63aaa0f7e4342d +size 915032 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o index 0380461ec..abf7ca1d1 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f41c44d5d523bbc37374ab62e91aaa0958d2faf1d44bec8635d0c7d8b625f3d7 -size 549888 +oid sha256:516ea3f7bc64ef0fdabd3aa49ed0378ec6e22b60d023e433a02861a79c9d7dea +size 550776 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o index f2d6358b8..06fd3cda9 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:38bcf808af9dd1fff7d0670425e2e155b391f908101e86931883b17fd94b27f4 -size 916064 +oid sha256:3db01edbc4fefb1e3101a382caedca4fd43cd5a99ecd4e3888fa621ee131473f +size 933304 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o index 2e6100264..d25a2eaba 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:af61a5c8ff1e1a7e119ebdf8089f40b19d7186b9d796049b2c61f5bbb3853b97 -size 915576 +oid sha256:dcd84ebf054f6faf7ffa9d5e8a48966251066b85df9326666f22a7b4f2d7d64d +size 932864 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o index ace917dc1..8f0939276 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:367c24125e88846560f792e060ac1e6b9ee28069b63e53af4f94533cf4580f35 -size 549296 +oid sha256:84124bcfa2d03c5f1512db17d0acbe9d158ce8e00ad403521acaeb60d0fa5701 +size 550176 diff --git a/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o index 464dfcd2a..68ec79306 100644 --- a/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7876d41ff32500f24006d0af1713900d1a77f8c021b39af986aec87074bca8e1 -size 534992 +oid sha256:d859ddef84f6eb1d9f144beaa69cb59fd08a96ae338b77d72f585c7cea4ce09d +size 535888 diff --git a/pkg/internal/ebpf/logger/logger.go b/pkg/internal/ebpf/logger/logger.go index 14a7c5e97..af8a60751 100644 --- a/pkg/internal/ebpf/logger/logger.go +++ b/pkg/internal/ebpf/logger/logger.go @@ -12,6 +12,7 @@ import ( "github.com/cilium/ebpf/ringbuf" "github.com/grafana/beyla/pkg/beyla" + "github.com/grafana/beyla/pkg/config" ebpfcommon "github.com/grafana/beyla/pkg/internal/ebpf/common" "github.com/grafana/beyla/pkg/internal/request" ) @@ -78,7 +79,7 @@ func (p *BPFLogger) Run(ctx context.Context) { )(ctx, nil) } -func (p *BPFLogger) processLogEvent(record *ringbuf.Record, _ ebpfcommon.ServiceFilter) (request.Span, bool, error) { +func (p *BPFLogger) processLogEvent(_ *config.EPPFTracer, record *ringbuf.Record, _ ebpfcommon.ServiceFilter) (request.Span, bool, error) { var event BPFLogInfo err := binary.Read(bytes.NewBuffer(record.RawSample), binary.LittleEndian, &event) diff --git a/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o index aa50b8f8a..bc5ee0d44 100644 --- a/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aabc72d22ca308a83838d368077ca0e16b2d87305bb5927b5caa8fdb204b1dd7 -size 206576 +oid sha256:ddf0d176b8e7f4c086403cd753222b633885ac1c37eaafcb9f29bc9c53da2495 +size 207288 diff --git a/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o index b336ba7fc..ba72bdedb 100644 --- a/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cf95ae239cf48cc25890ca6962aeb60dd3f4f159a967e2a0ba719079d6188638 -size 380088 +oid sha256:24dab2726f31be8cfc05b6f270a3e49735d12982fdffa9aa7179a3c81e534725 +size 380824 diff --git a/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o index 4ee60de35..c0c4516ee 100644 --- a/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0bcaf86afec020451e83116937754d4e71c6f9c3d018375744f257fa112b5fec -size 381400 +oid sha256:ead4c8e133887e4f71194770a8fb5004109701034d730a941b9e619d7d96ed8e +size 382128 diff --git a/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o index a52311403..8527be8f5 100644 --- a/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:930ca57b724e22d85cafe1f7df0e6aaa649a44dd9f5f6bdd291cd85f90e98b64 -size 207888 +oid sha256:54c47460fc90e3a103f6e35c5c7a694686dd8b0969be3768a9a4221839e4ac78 +size 208600 diff --git a/pkg/internal/ebpf/watcher/watcher.go b/pkg/internal/ebpf/watcher/watcher.go index 0a7d88703..bd68a0488 100644 --- a/pkg/internal/ebpf/watcher/watcher.go +++ b/pkg/internal/ebpf/watcher/watcher.go @@ -11,6 +11,7 @@ import ( "github.com/cilium/ebpf/ringbuf" "github.com/grafana/beyla/pkg/beyla" + "github.com/grafana/beyla/pkg/config" ebpfcommon "github.com/grafana/beyla/pkg/internal/ebpf/common" "github.com/grafana/beyla/pkg/internal/request" ) @@ -98,7 +99,7 @@ func (p *Watcher) Run(ctx context.Context) { )(ctx, nil) } -func (p *Watcher) processWatchEvent(record *ringbuf.Record, _ ebpfcommon.ServiceFilter) (request.Span, bool, error) { +func (p *Watcher) processWatchEvent(_ *config.EPPFTracer, record *ringbuf.Record, _ ebpfcommon.ServiceFilter) (request.Span, bool, error) { var flags uint64 var event BPFWatchInfo diff --git a/pkg/internal/request/span.go b/pkg/internal/request/span.go index e5c92eef1..c9ebd55aa 100644 --- a/pkg/internal/request/span.go +++ b/pkg/internal/request/span.go @@ -9,6 +9,7 @@ import ( "unicode/utf8" "github.com/gavv/monotime" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" semconv "go.opentelemetry.io/otel/semconv/v1.17.0" trace2 "go.opentelemetry.io/otel/trace" @@ -41,6 +42,14 @@ const ( grpcTracesDetectPattern = "/opentelemetry.proto.collector.trace.v1.TraceService/Export" ) +type SQLKind uint8 + +const ( + DBGeneric SQLKind = iota + 1 + DBPostgres + DBMySQL +) + func (t EventType) String() string { switch t { case EventTypeProcessAlive: @@ -148,6 +157,7 @@ type Span struct { HostName string `json:"hostName"` OtherNamespace string `json:"-"` Statement string `json:"-"` + SubType int `json:"-"` } func (s *Span) Inside(parent *Span) bool { @@ -497,3 +507,16 @@ func (s *Span) IsExportTracesSpan() bool { func (s *Span) IsSelfReferenceSpan() bool { return s.Peer == s.Host && (s.Service.UID.Namespace == s.OtherNamespace || s.OtherNamespace == "") } + +func (s *Span) DBSystem() attribute.KeyValue { + if s.Type == EventTypeSQLClient { + switch s.SubType { + case int(DBPostgres): + return semconv.DBSystemPostgreSQL + case int(DBMySQL): + return semconv.DBSystemMySQL + } + } + + return semconv.DBSystemOtherSQL +} diff --git a/pkg/internal/request/span_getters.go b/pkg/internal/request/span_getters.go index d5d4e59dc..dca14bfab 100644 --- a/pkg/internal/request/span_getters.go +++ b/pkg/internal/request/span_getters.go @@ -77,7 +77,7 @@ func SpanOTELGetters(name attr.Name) (attributes.Getter[*Span, attribute.KeyValu getter = func(span *Span) attribute.KeyValue { switch span.Type { case EventTypeSQLClient: - return DBSystem(semconv.DBSystemOtherSQL.Value.AsString()) + return DBSystem(span.DBSystem().Value.AsString()) case EventTypeRedisClient, EventTypeRedisServer: return DBSystem(semconv.DBSystemRedis.Value.AsString()) } @@ -149,7 +149,7 @@ func SpanPromGetters(attrName attr.Name) (attributes.Getter[*Span, string], bool getter = func(span *Span) string { switch span.Type { case EventTypeSQLClient: - return semconv.DBSystemOtherSQL.Value.AsString() + return span.DBSystem().Value.AsString() case EventTypeRedisClient, EventTypeRedisServer: return semconv.DBSystemRedis.Value.AsString() } @@ -158,7 +158,7 @@ func SpanPromGetters(attrName attr.Name) (attributes.Getter[*Span, string], bool case attr.DBCollectionName: getter = func(span *Span) string { if span.Type == EventTypeSQLClient { - return semconv.DBSystemOtherSQL.Value.AsString() + return span.DBSystem().Value.AsString() } return "" } diff --git a/test/integration/components/mysqldb/1-sakila-schema.sql b/test/integration/components/mysqldb/1-sakila-schema.sql new file mode 100644 index 000000000..4ade47139 --- /dev/null +++ b/test/integration/components/mysqldb/1-sakila-schema.sql @@ -0,0 +1,685 @@ +-- Sakila Sample Database Schema +-- Version 1.2 + +-- Copyright (c) 2006, 2019, Oracle and/or its affiliates. +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are +-- met: + +-- * Redistributions of source code must retain the above copyright notice, +-- this list of conditions and the following disclaimer. +-- * Redistributions in binary form must reproduce the above copyright +-- notice, this list of conditions and the following disclaimer in the +-- documentation and/or other materials provided with the distribution. +-- * Neither the name of Oracle nor the names of its contributors may be used +-- to endorse or promote products derived from this software without +-- specific prior written permission. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +-- IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +-- THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +-- PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +-- CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +-- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +-- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +-- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +-- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +-- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +-- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +SET NAMES utf8mb4; +SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; +SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; +SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL'; + +DROP SCHEMA IF EXISTS sakila; +CREATE SCHEMA sakila; +USE sakila; + +-- +-- Table structure for table `actor` +-- + +CREATE TABLE actor ( + actor_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, + first_name VARCHAR(45) NOT NULL, + last_name VARCHAR(45) NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (actor_id), + KEY idx_actor_last_name (last_name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `address` +-- + +CREATE TABLE address ( + address_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, + address VARCHAR(50) NOT NULL, + address2 VARCHAR(50) DEFAULT NULL, + district VARCHAR(20) NOT NULL, + city_id SMALLINT UNSIGNED NOT NULL, + postal_code VARCHAR(10) DEFAULT NULL, + phone VARCHAR(20) NOT NULL, + -- Add GEOMETRY column for MySQL 5.7.5 and higher + -- Also include SRID attribute for MySQL 8.0.3 and higher + /*!50705 location GEOMETRY */ /*!80003 SRID 0 */ /*!50705 NOT NULL,*/ + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (address_id), + KEY idx_fk_city_id (city_id), + /*!50705 SPATIAL KEY `idx_location` (location),*/ + CONSTRAINT `fk_address_city` FOREIGN KEY (city_id) REFERENCES city (city_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `category` +-- + +CREATE TABLE category ( + category_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, + name VARCHAR(25) NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (category_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `city` +-- + +CREATE TABLE city ( + city_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, + city VARCHAR(50) NOT NULL, + country_id SMALLINT UNSIGNED NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (city_id), + KEY idx_fk_country_id (country_id), + CONSTRAINT `fk_city_country` FOREIGN KEY (country_id) REFERENCES country (country_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `country` +-- + +CREATE TABLE country ( + country_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, + country VARCHAR(50) NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (country_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `customer` +-- + +CREATE TABLE customer ( + customer_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, + store_id TINYINT UNSIGNED NOT NULL, + first_name VARCHAR(45) NOT NULL, + last_name VARCHAR(45) NOT NULL, + email VARCHAR(50) DEFAULT NULL, + address_id SMALLINT UNSIGNED NOT NULL, + active BOOLEAN NOT NULL DEFAULT TRUE, + create_date DATETIME NOT NULL, + last_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (customer_id), + KEY idx_fk_store_id (store_id), + KEY idx_fk_address_id (address_id), + KEY idx_last_name (last_name), + CONSTRAINT fk_customer_address FOREIGN KEY (address_id) REFERENCES address (address_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_customer_store FOREIGN KEY (store_id) REFERENCES store (store_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `film` +-- + +CREATE TABLE film ( + film_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, + title VARCHAR(128) NOT NULL, + description TEXT DEFAULT NULL, + release_year YEAR DEFAULT NULL, + language_id TINYINT UNSIGNED NOT NULL, + original_language_id TINYINT UNSIGNED DEFAULT NULL, + rental_duration TINYINT UNSIGNED NOT NULL DEFAULT 3, + rental_rate DECIMAL(4,2) NOT NULL DEFAULT 4.99, + length SMALLINT UNSIGNED DEFAULT NULL, + replacement_cost DECIMAL(5,2) NOT NULL DEFAULT 19.99, + rating ENUM('G','PG','PG-13','R','NC-17') DEFAULT 'G', + special_features SET('Trailers','Commentaries','Deleted Scenes','Behind the Scenes') DEFAULT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (film_id), + KEY idx_title (title), + KEY idx_fk_language_id (language_id), + KEY idx_fk_original_language_id (original_language_id), + CONSTRAINT fk_film_language FOREIGN KEY (language_id) REFERENCES language (language_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_film_language_original FOREIGN KEY (original_language_id) REFERENCES language (language_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `film_actor` +-- + +CREATE TABLE film_actor ( + actor_id SMALLINT UNSIGNED NOT NULL, + film_id SMALLINT UNSIGNED NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (actor_id,film_id), + KEY idx_fk_film_id (`film_id`), + CONSTRAINT fk_film_actor_actor FOREIGN KEY (actor_id) REFERENCES actor (actor_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_film_actor_film FOREIGN KEY (film_id) REFERENCES film (film_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `film_category` +-- + +CREATE TABLE film_category ( + film_id SMALLINT UNSIGNED NOT NULL, + category_id TINYINT UNSIGNED NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (film_id, category_id), + CONSTRAINT fk_film_category_film FOREIGN KEY (film_id) REFERENCES film (film_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_film_category_category FOREIGN KEY (category_id) REFERENCES category (category_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `film_text` +-- +-- InnoDB added FULLTEXT support in 5.6.10. If you use an +-- earlier version, then consider upgrading (recommended) or +-- changing InnoDB to MyISAM as the film_text engine +-- + +-- Use InnoDB for film_text as of 5.6.10, MyISAM prior to 5.6.10. +SET @old_default_storage_engine = @@default_storage_engine; +SET @@default_storage_engine = 'MyISAM'; +/*!50610 SET @@default_storage_engine = 'InnoDB'*/; + +CREATE TABLE film_text ( + film_id SMALLINT NOT NULL, + title VARCHAR(255) NOT NULL, + description TEXT, + PRIMARY KEY (film_id), + FULLTEXT KEY idx_title_description (title,description) +) DEFAULT CHARSET=utf8mb4; + +SET @@default_storage_engine = @old_default_storage_engine; + +-- +-- Triggers for loading film_text from film +-- + +DELIMITER ;; +CREATE TRIGGER `ins_film` AFTER INSERT ON `film` FOR EACH ROW BEGIN + INSERT INTO film_text (film_id, title, description) + VALUES (new.film_id, new.title, new.description); + END;; + + +CREATE TRIGGER `upd_film` AFTER UPDATE ON `film` FOR EACH ROW BEGIN + IF (old.title != new.title) OR (old.description != new.description) OR (old.film_id != new.film_id) + THEN + UPDATE film_text + SET title=new.title, + description=new.description, + film_id=new.film_id + WHERE film_id=old.film_id; + END IF; + END;; + + +CREATE TRIGGER `del_film` AFTER DELETE ON `film` FOR EACH ROW BEGIN + DELETE FROM film_text WHERE film_id = old.film_id; + END;; + +DELIMITER ; + +-- +-- Table structure for table `inventory` +-- + +CREATE TABLE inventory ( + inventory_id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT, + film_id SMALLINT UNSIGNED NOT NULL, + store_id TINYINT UNSIGNED NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (inventory_id), + KEY idx_fk_film_id (film_id), + KEY idx_store_id_film_id (store_id,film_id), + CONSTRAINT fk_inventory_store FOREIGN KEY (store_id) REFERENCES store (store_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_inventory_film FOREIGN KEY (film_id) REFERENCES film (film_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `language` +-- + +CREATE TABLE language ( + language_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, + name CHAR(20) NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (language_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `payment` +-- + +CREATE TABLE payment ( + payment_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, + customer_id SMALLINT UNSIGNED NOT NULL, + staff_id TINYINT UNSIGNED NOT NULL, + rental_id INT DEFAULT NULL, + amount DECIMAL(5,2) NOT NULL, + payment_date DATETIME NOT NULL, + last_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (payment_id), + KEY idx_fk_staff_id (staff_id), + KEY idx_fk_customer_id (customer_id), + CONSTRAINT fk_payment_rental FOREIGN KEY (rental_id) REFERENCES rental (rental_id) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT fk_payment_customer FOREIGN KEY (customer_id) REFERENCES customer (customer_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_payment_staff FOREIGN KEY (staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +-- +-- Table structure for table `rental` +-- + +CREATE TABLE rental ( + rental_id INT NOT NULL AUTO_INCREMENT, + rental_date DATETIME NOT NULL, + inventory_id MEDIUMINT UNSIGNED NOT NULL, + customer_id SMALLINT UNSIGNED NOT NULL, + return_date DATETIME DEFAULT NULL, + staff_id TINYINT UNSIGNED NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (rental_id), + UNIQUE KEY (rental_date,inventory_id,customer_id), + KEY idx_fk_inventory_id (inventory_id), + KEY idx_fk_customer_id (customer_id), + KEY idx_fk_staff_id (staff_id), + CONSTRAINT fk_rental_staff FOREIGN KEY (staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_rental_inventory FOREIGN KEY (inventory_id) REFERENCES inventory (inventory_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_rental_customer FOREIGN KEY (customer_id) REFERENCES customer (customer_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `staff` +-- + +CREATE TABLE staff ( + staff_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, + first_name VARCHAR(45) NOT NULL, + last_name VARCHAR(45) NOT NULL, + address_id SMALLINT UNSIGNED NOT NULL, + picture BLOB DEFAULT NULL, + email VARCHAR(50) DEFAULT NULL, + store_id TINYINT UNSIGNED NOT NULL, + active BOOLEAN NOT NULL DEFAULT TRUE, + username VARCHAR(16) NOT NULL, + password VARCHAR(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (staff_id), + KEY idx_fk_store_id (store_id), + KEY idx_fk_address_id (address_id), + CONSTRAINT fk_staff_store FOREIGN KEY (store_id) REFERENCES store (store_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_staff_address FOREIGN KEY (address_id) REFERENCES address (address_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- Table structure for table `store` +-- + +CREATE TABLE store ( + store_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, + manager_staff_id TINYINT UNSIGNED NOT NULL, + address_id SMALLINT UNSIGNED NOT NULL, + last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (store_id), + UNIQUE KEY idx_unique_manager (manager_staff_id), + KEY idx_fk_address_id (address_id), + CONSTRAINT fk_store_staff FOREIGN KEY (manager_staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT fk_store_address FOREIGN KEY (address_id) REFERENCES address (address_id) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- +-- View structure for view `customer_list` +-- + +CREATE VIEW customer_list +AS +SELECT cu.customer_id AS ID, CONCAT(cu.first_name, _utf8mb4' ', cu.last_name) AS name, a.address AS address, a.postal_code AS `zip code`, + a.phone AS phone, city.city AS city, country.country AS country, IF(cu.active, _utf8mb4'active',_utf8mb4'') AS notes, cu.store_id AS SID +FROM customer AS cu JOIN address AS a ON cu.address_id = a.address_id JOIN city ON a.city_id = city.city_id + JOIN country ON city.country_id = country.country_id; + +-- +-- View structure for view `film_list` +-- + +CREATE VIEW film_list +AS +SELECT film.film_id AS FID, film.title AS title, film.description AS description, category.name AS category, film.rental_rate AS price, + film.length AS length, film.rating AS rating, GROUP_CONCAT(CONCAT(actor.first_name, _utf8mb4' ', actor.last_name) SEPARATOR ', ') AS actors +FROM category LEFT JOIN film_category ON category.category_id = film_category.category_id LEFT JOIN film ON film_category.film_id = film.film_id + JOIN film_actor ON film.film_id = film_actor.film_id + JOIN actor ON film_actor.actor_id = actor.actor_id +GROUP BY film.film_id, category.name; + +-- +-- View structure for view `nicer_but_slower_film_list` +-- + +CREATE VIEW nicer_but_slower_film_list +AS +SELECT film.film_id AS FID, film.title AS title, film.description AS description, category.name AS category, film.rental_rate AS price, + film.length AS length, film.rating AS rating, GROUP_CONCAT(CONCAT(CONCAT(UCASE(SUBSTR(actor.first_name,1,1)), + LCASE(SUBSTR(actor.first_name,2,LENGTH(actor.first_name))),_utf8mb4' ',CONCAT(UCASE(SUBSTR(actor.last_name,1,1)), + LCASE(SUBSTR(actor.last_name,2,LENGTH(actor.last_name)))))) SEPARATOR ', ') AS actors +FROM category LEFT JOIN film_category ON category.category_id = film_category.category_id LEFT JOIN film ON film_category.film_id = film.film_id + JOIN film_actor ON film.film_id = film_actor.film_id + JOIN actor ON film_actor.actor_id = actor.actor_id +GROUP BY film.film_id, category.name; + +-- +-- View structure for view `staff_list` +-- + +CREATE VIEW staff_list +AS +SELECT s.staff_id AS ID, CONCAT(s.first_name, _utf8mb4' ', s.last_name) AS name, a.address AS address, a.postal_code AS `zip code`, a.phone AS phone, + city.city AS city, country.country AS country, s.store_id AS SID +FROM staff AS s JOIN address AS a ON s.address_id = a.address_id JOIN city ON a.city_id = city.city_id + JOIN country ON city.country_id = country.country_id; + +-- +-- View structure for view `sales_by_store` +-- + +CREATE VIEW sales_by_store +AS +SELECT +CONCAT(c.city, _utf8mb4',', cy.country) AS store +, CONCAT(m.first_name, _utf8mb4' ', m.last_name) AS manager +, SUM(p.amount) AS total_sales +FROM payment AS p +INNER JOIN rental AS r ON p.rental_id = r.rental_id +INNER JOIN inventory AS i ON r.inventory_id = i.inventory_id +INNER JOIN store AS s ON i.store_id = s.store_id +INNER JOIN address AS a ON s.address_id = a.address_id +INNER JOIN city AS c ON a.city_id = c.city_id +INNER JOIN country AS cy ON c.country_id = cy.country_id +INNER JOIN staff AS m ON s.manager_staff_id = m.staff_id +GROUP BY s.store_id +ORDER BY cy.country, c.city; + +-- +-- View structure for view `sales_by_film_category` +-- +-- Note that total sales will add up to >100% because +-- some titles belong to more than 1 category +-- + +CREATE VIEW sales_by_film_category +AS +SELECT +c.name AS category +, SUM(p.amount) AS total_sales +FROM payment AS p +INNER JOIN rental AS r ON p.rental_id = r.rental_id +INNER JOIN inventory AS i ON r.inventory_id = i.inventory_id +INNER JOIN film AS f ON i.film_id = f.film_id +INNER JOIN film_category AS fc ON f.film_id = fc.film_id +INNER JOIN category AS c ON fc.category_id = c.category_id +GROUP BY c.name +ORDER BY total_sales DESC; + +-- +-- View structure for view `actor_info` +-- + +CREATE DEFINER=CURRENT_USER SQL SECURITY INVOKER VIEW actor_info +AS +SELECT +a.actor_id, +a.first_name, +a.last_name, +GROUP_CONCAT(DISTINCT CONCAT(c.name, ': ', + (SELECT GROUP_CONCAT(f.title ORDER BY f.title SEPARATOR ', ') + FROM sakila.film f + INNER JOIN sakila.film_category fc + ON f.film_id = fc.film_id + INNER JOIN sakila.film_actor fa + ON f.film_id = fa.film_id + WHERE fc.category_id = c.category_id + AND fa.actor_id = a.actor_id + ) + ) + ORDER BY c.name SEPARATOR '; ') +AS film_info +FROM sakila.actor a +LEFT JOIN sakila.film_actor fa + ON a.actor_id = fa.actor_id +LEFT JOIN sakila.film_category fc + ON fa.film_id = fc.film_id +LEFT JOIN sakila.category c + ON fc.category_id = c.category_id +GROUP BY a.actor_id, a.first_name, a.last_name; + +-- +-- Procedure structure for procedure `rewards_report` +-- + +DELIMITER // + +CREATE PROCEDURE rewards_report ( + IN min_monthly_purchases TINYINT UNSIGNED + , IN min_dollar_amount_purchased DECIMAL(10,2) + , OUT count_rewardees INT +) +LANGUAGE SQL +NOT DETERMINISTIC +READS SQL DATA +SQL SECURITY DEFINER +COMMENT 'Provides a customizable report on best customers' +proc: BEGIN + + DECLARE last_month_start DATE; + DECLARE last_month_end DATE; + + /* Some sanity checks... */ + IF min_monthly_purchases = 0 THEN + SELECT 'Minimum monthly purchases parameter must be > 0'; + LEAVE proc; + END IF; + IF min_dollar_amount_purchased = 0.00 THEN + SELECT 'Minimum monthly dollar amount purchased parameter must be > $0.00'; + LEAVE proc; + END IF; + + /* Determine start and end time periods */ + SET last_month_start = DATE_SUB(CURRENT_DATE(), INTERVAL 1 MONTH); + SET last_month_start = STR_TO_DATE(CONCAT(YEAR(last_month_start),'-',MONTH(last_month_start),'-01'),'%Y-%m-%d'); + SET last_month_end = LAST_DAY(last_month_start); + + /* + Create a temporary storage area for + Customer IDs. + */ + CREATE TEMPORARY TABLE tmpCustomer (customer_id SMALLINT UNSIGNED NOT NULL PRIMARY KEY); + + /* + Find all customers meeting the + monthly purchase requirements + */ + INSERT INTO tmpCustomer (customer_id) + SELECT p.customer_id + FROM payment AS p + WHERE DATE(p.payment_date) BETWEEN last_month_start AND last_month_end + GROUP BY customer_id + HAVING SUM(p.amount) > min_dollar_amount_purchased + AND COUNT(customer_id) > min_monthly_purchases; + + /* Populate OUT parameter with count of found customers */ + SELECT COUNT(*) FROM tmpCustomer INTO count_rewardees; + + /* + Output ALL customer information of matching rewardees. + Customize output as needed. + */ + SELECT c.* + FROM tmpCustomer AS t + INNER JOIN customer AS c ON t.customer_id = c.customer_id; + + /* Clean up */ + DROP TABLE tmpCustomer; +END // + +DELIMITER ; + +DELIMITER $$ + +CREATE FUNCTION get_customer_balance(p_customer_id INT, p_effective_date DATETIME) RETURNS DECIMAL(5,2) + DETERMINISTIC + READS SQL DATA +BEGIN + + #OK, WE NEED TO CALCULATE THE CURRENT BALANCE GIVEN A CUSTOMER_ID AND A DATE + #THAT WE WANT THE BALANCE TO BE EFFECTIVE FOR. THE BALANCE IS: + # 1) RENTAL FEES FOR ALL PREVIOUS RENTALS + # 2) ONE DOLLAR FOR EVERY DAY THE PREVIOUS RENTALS ARE OVERDUE + # 3) IF A FILM IS MORE THAN RENTAL_DURATION * 2 OVERDUE, CHARGE THE REPLACEMENT_COST + # 4) SUBTRACT ALL PAYMENTS MADE BEFORE THE DATE SPECIFIED + + DECLARE v_rentfees DECIMAL(5,2); #FEES PAID TO RENT THE VIDEOS INITIALLY + DECLARE v_overfees INTEGER; #LATE FEES FOR PRIOR RENTALS + DECLARE v_payments DECIMAL(5,2); #SUM OF PAYMENTS MADE PREVIOUSLY + + SELECT IFNULL(SUM(film.rental_rate),0) INTO v_rentfees + FROM film, inventory, rental + WHERE film.film_id = inventory.film_id + AND inventory.inventory_id = rental.inventory_id + AND rental.rental_date <= p_effective_date + AND rental.customer_id = p_customer_id; + + SELECT IFNULL(SUM(IF((TO_DAYS(rental.return_date) - TO_DAYS(rental.rental_date)) > film.rental_duration, + ((TO_DAYS(rental.return_date) - TO_DAYS(rental.rental_date)) - film.rental_duration),0)),0) INTO v_overfees + FROM rental, inventory, film + WHERE film.film_id = inventory.film_id + AND inventory.inventory_id = rental.inventory_id + AND rental.rental_date <= p_effective_date + AND rental.customer_id = p_customer_id; + + + SELECT IFNULL(SUM(payment.amount),0) INTO v_payments + FROM payment + + WHERE payment.payment_date <= p_effective_date + AND payment.customer_id = p_customer_id; + + RETURN v_rentfees + v_overfees - v_payments; +END $$ + +DELIMITER ; + +DELIMITER $$ + +CREATE PROCEDURE film_in_stock(IN p_film_id INT, IN p_store_id INT, OUT p_film_count INT) +READS SQL DATA +BEGIN + SELECT inventory_id + FROM inventory + WHERE film_id = p_film_id + AND store_id = p_store_id + AND inventory_in_stock(inventory_id); + + SELECT COUNT(*) + FROM inventory + WHERE film_id = p_film_id + AND store_id = p_store_id + AND inventory_in_stock(inventory_id) + INTO p_film_count; +END $$ + +DELIMITER ; + +DELIMITER $$ + +CREATE PROCEDURE film_not_in_stock(IN p_film_id INT, IN p_store_id INT, OUT p_film_count INT) +READS SQL DATA +BEGIN + SELECT inventory_id + FROM inventory + WHERE film_id = p_film_id + AND store_id = p_store_id + AND NOT inventory_in_stock(inventory_id); + + SELECT COUNT(*) + FROM inventory + WHERE film_id = p_film_id + AND store_id = p_store_id + AND NOT inventory_in_stock(inventory_id) + INTO p_film_count; +END $$ + +DELIMITER ; + +DELIMITER $$ + +CREATE FUNCTION inventory_held_by_customer(p_inventory_id INT) RETURNS INT +READS SQL DATA +BEGIN + DECLARE v_customer_id INT; + DECLARE EXIT HANDLER FOR NOT FOUND RETURN NULL; + + SELECT customer_id INTO v_customer_id + FROM rental + WHERE return_date IS NULL + AND inventory_id = p_inventory_id; + + RETURN v_customer_id; +END $$ + +DELIMITER ; + +DELIMITER $$ + +CREATE FUNCTION inventory_in_stock(p_inventory_id INT) RETURNS BOOLEAN +READS SQL DATA +BEGIN + DECLARE v_rentals INT; + DECLARE v_out INT; + + #AN ITEM IS IN-STOCK IF THERE ARE EITHER NO ROWS IN THE rental TABLE + #FOR THE ITEM OR ALL ROWS HAVE return_date POPULATED + + SELECT COUNT(*) INTO v_rentals + FROM rental + WHERE inventory_id = p_inventory_id; + + IF v_rentals = 0 THEN + RETURN TRUE; + END IF; + + SELECT COUNT(rental_id) INTO v_out + FROM inventory LEFT JOIN rental USING(inventory_id) + WHERE inventory.inventory_id = p_inventory_id + AND rental.return_date IS NULL; + + IF v_out > 0 THEN + RETURN FALSE; + ELSE + RETURN TRUE; + END IF; +END $$ + +DELIMITER ; + +SET SQL_MODE=@OLD_SQL_MODE; +SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; +SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; + + diff --git a/test/integration/components/mysqldb/2-sakila-data.sql b/test/integration/components/mysqldb/2-sakila-data.sql new file mode 100644 index 000000000..500e2d99e --- /dev/null +++ b/test/integration/components/mysqldb/2-sakila-data.sql @@ -0,0 +1,254 @@ +-- Sakila Sample Database Data +-- Version 1.2 + +-- Copyright (c) 2006, 2019, Oracle and/or its affiliates. +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are +-- met: + +-- * Redistributions of source code must retain the above copyright notice, +-- this list of conditions and the following disclaimer. +-- * Redistributions in binary form must reproduce the above copyright +-- notice, this list of conditions and the following disclaimer in the +-- documentation and/or other materials provided with the distribution. +-- * Neither the name of Oracle nor the names of its contributors may be used +-- to endorse or promote products derived from this software without +-- specific prior written permission. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +-- IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +-- THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +-- PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +-- CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +-- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +-- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +-- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +-- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +-- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +-- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +SET NAMES utf8mb4; +SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; +SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; +SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL'; +SET @old_autocommit=@@autocommit; + +USE sakila; + +-- +-- Dumping data for table actor +-- + +SET AUTOCOMMIT=0; +INSERT INTO actor VALUES (1,'PENELOPE','GUINESS','2006-02-15 04:34:33'), +(2,'NICK','WAHLBERG','2006-02-15 04:34:33'), +(3,'ED','CHASE','2006-02-15 04:34:33'), +(4,'JENNIFER','DAVIS','2006-02-15 04:34:33'), +(5,'JOHNNY','LOLLOBRIGIDA','2006-02-15 04:34:33'), +(6,'BETTE','NICHOLSON','2006-02-15 04:34:33'), +(7,'GRACE','MOSTEL','2006-02-15 04:34:33'), +(8,'MATTHEW','JOHANSSON','2006-02-15 04:34:33'); +COMMIT; + +-- +-- Dumping data for table address +-- + +SET AUTOCOMMIT=0; +INSERT INTO `address` VALUES (1,'47 MySakila Drive',NULL,'Alberta',300,'','',/*!50705 0x0000000001010000003E0A325D63345CC0761FDB8D99D94840,*/'2014-09-25 22:30:27'), +(2,'28 MySQL Boulevard',NULL,'QLD',576,'','',/*!50705 0x0000000001010000008E10D4DF812463404EE08C5022A23BC0,*/'2014-09-25 22:30:09'), +(3,'23 Workhaven Lane',NULL,'Alberta',300,'','14033335568',/*!50705 0x000000000101000000CDC4196863345CC01DEE7E7099D94840,*/'2014-09-25 22:30:27'), +(4,'1411 Lillydale Drive',NULL,'QLD',576,'','6172235589',/*!50705 0x0000000001010000005B0DE4341F26634042D6AE6422A23BC0,*/'2014-09-25 22:30:09'), +(5,'1913 Hanoi Way','','Nagasaki',463,'35200','28303384290',/*!50705 0x00000000010100000028D1370E21376040ABB58BC45F944040,*/'2014-09-25 22:31:53'); +COMMIT; + +-- +-- Dumping data for table category +-- + +SET AUTOCOMMIT=0; +INSERT INTO category VALUES (1,'Action','2006-02-15 04:46:27'), +(2,'Animation','2006-02-15 04:46:27'), +(3,'Children','2006-02-15 04:46:27'), +(4,'Classics','2006-02-15 04:46:27'), +(5,'Comedy','2006-02-15 04:46:27'), +(6,'Documentary','2006-02-15 04:46:27'), +(7,'Drama','2006-02-15 04:46:27'), +(8,'Family','2006-02-15 04:46:27'), +(9,'Foreign','2006-02-15 04:46:27'), +(10,'Games','2006-02-15 04:46:27'), +(11,'Horror','2006-02-15 04:46:27'), +(12,'Music','2006-02-15 04:46:27'), +(13,'New','2006-02-15 04:46:27'), +(14,'Sci-Fi','2006-02-15 04:46:27'), +(15,'Sports','2006-02-15 04:46:27'), +(16,'Travel','2006-02-15 04:46:27'); +COMMIT; + +-- +-- Dumping data for table city +-- + +SET AUTOCOMMIT=0; +INSERT INTO city VALUES (1,'A Corua (La Corua)',87,'2006-02-15 04:45:25'), +(2,'Abha',82,'2006-02-15 04:45:25'), +(3,'Abu Dhabi',101,'2006-02-15 04:45:25'), +(4,'Acua',60,'2006-02-15 04:45:25'), +(5,'Adana',97,'2006-02-15 04:45:25'), +(6,'Addis Abeba',31,'2006-02-15 04:45:25'); +COMMIT; + +-- +-- Dumping data for table country +-- + +SET AUTOCOMMIT=0; +INSERT INTO country VALUES (1,'Afghanistan','2006-02-15 04:44:00'), +(2,'Algeria','2006-02-15 04:44:00'), +(3,'American Samoa','2006-02-15 04:44:00'), +(4,'Angola','2006-02-15 04:44:00'), +(5,'Anguilla','2006-02-15 04:44:00'), +(6,'Argentina','2006-02-15 04:44:00'), +(7,'Armenia','2006-02-15 04:44:00'); +COMMIT; + +-- +-- Dumping data for table customer +-- + +SET AUTOCOMMIT=0; +INSERT INTO customer VALUES (1,1,'MARY','SMITH','MARY.SMITH@sakilacustomer.org',5,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), +(2,1,'PATRICIA','JOHNSON','PATRICIA.JOHNSON@sakilacustomer.org',6,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), +(3,1,'LINDA','WILLIAMS','LINDA.WILLIAMS@sakilacustomer.org',7,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), +(4,2,'BARBARA','JONES','BARBARA.JONES@sakilacustomer.org',8,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), +(5,1,'ELIZABETH','BROWN','ELIZABETH.BROWN@sakilacustomer.org',9,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), +(6,2,'JENNIFER','DAVIS','JENNIFER.DAVIS@sakilacustomer.org',10,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), +(7,1,'MARIA','MILLER','MARIA.MILLER@sakilacustomer.org',11,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), +(8,2,'SUSAN','WILSON','SUSAN.WILSON@sakilacustomer.org',12,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'); +COMMIT; + +-- +-- Trigger to enforce create dates on INSERT +-- + +CREATE TRIGGER customer_create_date BEFORE INSERT ON customer + FOR EACH ROW SET NEW.create_date = NOW(); + +-- +-- Dumping data for table film +-- + +SET AUTOCOMMIT=0; +INSERT INTO film VALUES (1,'ACADEMY DINOSAUR','A Epic Drama of a Feminist And a Mad Scientist who must Battle a Teacher in The Canadian Rockies',2006,1,NULL,6,'0.99',86,'20.99','PG','Deleted Scenes,Behind the Scenes','2006-02-15 05:03:42'), +(2,'ACE GOLDFINGER','A Astounding Epistle of a Database Administrator And a Explorer who must Find a Car in Ancient China',2006,1,NULL,3,'4.99',48,'12.99','G','Trailers,Deleted Scenes','2006-02-15 05:03:42'), +(3,'ADAPTATION HOLES','A Astounding Reflection of a Lumberjack And a Car who must Sink a Lumberjack in A Baloon Factory',2006,1,NULL,7,'2.99',50,'18.99','NC-17','Trailers,Deleted Scenes','2006-02-15 05:03:42'), +(4,'AFFAIR PREJUDICE','A Fanciful Documentary of a Frisbee And a Lumberjack who must Chase a Monkey in A Shark Tank',2006,1,NULL,5,'2.99',117,'26.99','G','Commentaries,Behind the Scenes','2006-02-15 05:03:42'), +(5,'AFRICAN EGG','A Fast-Paced Documentary of a Pastry Chef And a Dentist who must Pursue a Forensic Psychologist in The Gulf of Mexico',2006,1,NULL,6,'2.99',130,'22.99','G','Deleted Scenes','2006-02-15 05:03:42'), +(6,'AGENT TRUMAN','A Intrepid Panorama of a Robot And a Boy who must Escape a Sumo Wrestler in Ancient China',2006,1,NULL,3,'2.99',169,'17.99','PG','Deleted Scenes','2006-02-15 05:03:42'), +(7,'AIRPLANE SIERRA','A Touching Saga of a Hunter And a Butler who must Discover a Butler in A Jet Boat',2006,1,NULL,6,'4.99',62,'28.99','PG-13','Trailers,Deleted Scenes','2006-02-15 05:03:42'), +(8,'AIRPORT POLLOCK','A Epic Tale of a Moose And a Girl who must Confront a Monkey in Ancient India',2006,1,NULL,6,'4.99',54,'15.99','R','Trailers','2006-02-15 05:03:42'); +COMMIT; + +-- +-- Dumping data for table film_actor +-- + +SET AUTOCOMMIT=0; +INSERT INTO film_actor VALUES (1,1,'2006-02-15 05:05:03'), +(1,23,'2006-02-15 05:05:03'), +(1,25,'2006-02-15 05:05:03'), +(1,106,'2006-02-15 05:05:03'), +(1,140,'2006-02-15 05:05:03'), +(1,166,'2006-02-15 05:05:03'), +(1,277,'2006-02-15 05:05:03'), +(1,361,'2006-02-15 05:05:03'), +(1,438,'2006-02-15 05:05:03'); +COMMIT; + +-- +-- Dumping data for table film_category +-- + +SET AUTOCOMMIT=0; +INSERT INTO film_category VALUES (1,6,'2006-02-15 05:07:09'), +(2,11,'2006-02-15 05:07:09'), +(3,6,'2006-02-15 05:07:09'), +(4,11,'2006-02-15 05:07:09'), +(5,8,'2006-02-15 05:07:09'), +(6,9,'2006-02-15 05:07:09'), +(7,5,'2006-02-15 05:07:09'), +(8,11,'2006-02-15 05:07:09'); +COMMIT; + +-- +-- Dumping data for table language +-- + +SET AUTOCOMMIT=0; +INSERT INTO language VALUES (1,'English','2006-02-15 05:02:19'), +(2,'Italian','2006-02-15 05:02:19'), +(3,'Japanese','2006-02-15 05:02:19'), +(4,'Mandarin','2006-02-15 05:02:19'), +(5,'French','2006-02-15 05:02:19'), +(6,'German','2006-02-15 05:02:19'); +COMMIT; + +-- +-- Dumping data for table payment +-- + +SET AUTOCOMMIT=0; +INSERT INTO payment VALUES (1,1,1,76,'2.99','2005-05-25 11:30:37','2006-02-15 22:12:30'), +(2,1,1,573,'0.99','2005-05-28 10:35:23','2006-02-15 22:12:30'), +(3,1,1,1185,'5.99','2005-06-15 00:54:12','2006-02-15 22:12:30'), +(4,1,2,1422,'0.99','2005-06-15 18:02:53','2006-02-15 22:12:30'), +(5,1,2,1476,'9.99','2005-06-15 21:08:46','2006-02-15 22:12:30'), +(6,1,1,1725,'4.99','2005-06-16 15:18:57','2006-02-15 22:12:30'), +(7,1,1,2308,'4.99','2005-06-18 08:41:48','2006-02-15 22:12:30'), +(8,1,2,2363,'0.99','2005-06-18 13:33:59','2006-02-15 22:12:30'); +COMMIT; + +-- +-- Trigger to enforce payment_date during INSERT +-- + +CREATE TRIGGER payment_date BEFORE INSERT ON payment + FOR EACH ROW SET NEW.payment_date = NOW(); + +-- +-- Dumping data for table rental +-- + +SET AUTOCOMMIT=0; +INSERT INTO rental VALUES (1,'2005-05-24 22:53:30',367,130,'2005-05-26 22:04:30',1,'2006-02-15 21:30:53'), +(2,'2005-05-24 22:54:33',1525,459,'2005-05-28 19:40:33',1,'2006-02-15 21:30:53'), +(3,'2005-05-24 23:03:39',1711,408,'2005-06-01 22:12:39',1,'2006-02-15 21:30:53'), +(4,'2005-05-24 23:04:41',2452,333,'2005-06-03 01:43:41',2,'2006-02-15 21:30:53'), +(5,'2005-05-24 23:05:21',2079,222,'2005-06-02 04:33:21',1,'2006-02-15 21:30:53'), +(6,'2005-05-24 23:08:07',2792,549,'2005-05-27 01:32:07',1,'2006-02-15 21:30:53'), +(7,'2005-05-24 23:11:53',3995,269,'2005-05-29 20:34:53',2,'2006-02-15 21:30:53'), +(8,'2005-05-24 23:31:46',2346,239,'2005-05-27 23:33:46',2,'2006-02-15 21:30:53'); +COMMIT; + +-- +-- Trigger to enforce rental_date on INSERT +-- + +CREATE TRIGGER rental_date BEFORE INSERT ON rental + FOR EACH ROW SET NEW.rental_date = NOW(); + +-- +-- Dumping data for table store +-- + +SET AUTOCOMMIT=0; +INSERT INTO store VALUES (1,1,1,'2006-02-15 04:57:12'), +(2,2,2,'2006-02-15 04:57:12'); +COMMIT; + +SET SQL_MODE=@OLD_SQL_MODE; +SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; +SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; +SET autocommit=@old_autocommit; diff --git a/test/integration/components/mysqldb/3-sakila-complete.sql b/test/integration/components/mysqldb/3-sakila-complete.sql new file mode 100644 index 000000000..fdedc4e5a --- /dev/null +++ b/test/integration/components/mysqldb/3-sakila-complete.sql @@ -0,0 +1,4 @@ +GRANT ALL PRIVILEGES ON *.* TO 'sakila'@'%' WITH GRANT OPTION; +FLUSH PRIVILEGES; + +SELECT 'sakiladb/mysql has successfully initialized.' AS sakiladb_completion_message; diff --git a/test/integration/components/mysqldb/Dockerfile b/test/integration/components/mysqldb/Dockerfile new file mode 100644 index 000000000..db7dcae6d --- /dev/null +++ b/test/integration/components/mysqldb/Dockerfile @@ -0,0 +1,33 @@ +FROM mysql:8 as builder +ENV MYSQL_ROOT_PASSWORD=p_ssW0rd +ENV MYSQL_DATABASE=sakila +ENV MYSQL_USER=sakila +ENV MYSQL_PASSWORD=p_ssW0rd + +COPY ./1-sakila-schema.sql /docker-entrypoint-initdb.d/step_1.sql +COPY ./2-sakila-data.sql /docker-entrypoint-initdb.d/step_2.sql +COPY ./3-sakila-complete.sql /docker-entrypoint-initdb.d/step_3.sql + +# https://serverfault.com/questions/930141/creating-a-mysql-image-with-the-db-preloaded +# https://serverfault.com/questions/796762/creating-a-docker-mysql-container-with-a-prepared-database-scheme +RUN ["sed", "-i", "s/exec \"$@\"/echo \"skipping...\"/", "/usr/local/bin/docker-entrypoint.sh"] + +USER mysql +RUN ["/usr/local/bin/docker-entrypoint.sh", "mysqld"] + +FROM mysql:8 +ENV MYSQL_ROOT_PASSWORD=p_ssW0rd +ENV MYSQL_DATABASE=sakila +ENV MYSQL_USER=sakila +ENV MYSQL_PASSWORD=p_ssW0rd + +COPY --from=builder /var/lib/mysql /data +RUN rm -rf /var/lib/mysql/* +RUN mv /data/* /var/lib/mysql/ + +USER mysql + +# See: https://dev.to/mdemblani/docker-container-uncaught-kill-signal-10l6 +COPY ./signal-listener.sh /sakila/run.sh +# Entrypoint overload to catch the ctrl+c and stop signals +ENTRYPOINT ["/bin/bash", "/sakila/run.sh"] diff --git a/test/integration/components/mysqldb/LICENSE b/test/integration/components/mysqldb/LICENSE new file mode 100644 index 000000000..d4f24dd49 --- /dev/null +++ b/test/integration/components/mysqldb/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020, Sakila DB +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/test/integration/components/mysqldb/signal-listener.sh b/test/integration/components/mysqldb/signal-listener.sh new file mode 100755 index 000000000..b8952970b --- /dev/null +++ b/test/integration/components/mysqldb/signal-listener.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# A wrapper around docker-entrypoint.sh to trap the SIGINT signal (Ctrl+C) and forwards it to the mysql daemon +# In other words : traps SIGINT and SIGTERM signals and forwards them to the child process as SIGTERM signals + +# See https://dev.to/mdemblani/docker-container-uncaught-kill-signal-10l6 + +signalListener() { + "$@" & + pid="$!" + trap "echo 'Stopping PID $pid'; kill -SIGTERM $pid" SIGINT SIGTERM + + # A signal emitted while waiting will make the wait command return code > 128 + # Let's wrap it in a loop that doesn't end before the process is indeed stopped + while kill -0 $pid > /dev/null 2>&1; do + wait + done +} + +signalListener /usr/local/bin/docker-entrypoint.sh mysqld diff --git a/test/integration/components/pythonsql/Dockerfile_mysql b/test/integration/components/pythonsql/Dockerfile_mysql new file mode 100644 index 000000000..e355a47aa --- /dev/null +++ b/test/integration/components/pythonsql/Dockerfile_mysql @@ -0,0 +1,7 @@ +# Dockerfile that will build a container that runs python with FastAPI and uvicorn on port 8080 +FROM python:3.12 +EXPOSE 8080 +RUN apt update +RUN pip install fastapi uvicorn mysql-connector-python +COPY main_mysql.py /main.py +CMD ["uvicorn", "--port", "8080", "--host", "0.0.0.0", "main:app"] \ No newline at end of file diff --git a/test/integration/components/pythonsql/main_mysql.py b/test/integration/components/pythonsql/main_mysql.py new file mode 100644 index 000000000..8e5091e1a --- /dev/null +++ b/test/integration/components/pythonsql/main_mysql.py @@ -0,0 +1,77 @@ +from fastapi import FastAPI +import os +import uvicorn +import mysql.connector + +app = FastAPI() + +conn = None + +@app.get("/query") +async def root(): + global conn + if conn is None: + conn = mysql.connector.connect( + database="sakila", + user="sakila", + password="p_ssW0rd", + host="sqlserver", + port="3306" + ) + + cur = conn.cursor() + cur.execute("SELECT * from actor WHERE actor_id=1") + + row = cur.fetchone() + + return row + +@app.get("/argquery") +async def root(): + global conn + if conn is None: + conn = mysql.connector.connect( + database="sakila", + user="sakila", + password="p_ssW0rd", + host="sqlserver", + port="3306" + ) + + cur = conn.cursor() + cur.execute("SELECT * from actor WHERE actor_id=%s", [1]) + + row = cur.fetchone() + + return row + +gCurr = None + +@app.get("/prepquery") +async def root(): + global conn + global gCurr + if conn is None: + conn = mysql.connector.connect( + database="sakila", + user="sakila", + password="p_ssW0rd", + host="sqlserver", + port="3306" + ) + + if gCurr is None: + gCurr = conn.cursor() + gCurr.execute( + "PREPARE my_actors FROM 'SELECT * FROM actor WHERE actor_id = ?'" + ) + + gCurr.execute("EXECUTE my_actors USING @actor_id", {'actor_id': 1}) + + row = gCurr.fetchone() + + return row + +if __name__ == "__main__": + print(f"Server running: port={8080} process_id={os.getpid()}") + uvicorn.run(app, host="0.0.0.0", port=8080) \ No newline at end of file diff --git a/test/integration/test_utils.go b/test/integration/test_utils.go index 47012d38f..1e40079ce 100644 --- a/test/integration/test_utils.go +++ b/test/integration/test_utils.go @@ -168,7 +168,7 @@ func waitForSQLTestComponents(t *testing.T, url, subpath string) { // now, verify that the metric has been reported. // we don't really care that this metric could be from a previous // test. Once one it is visible, it means that Otel and Prometheus are healthy - results, err := pq.Query(`db_client_operation_duration_seconds_count{db_system="other_sql"}`) + results, err := pq.Query(`db_client_operation_duration_seconds_count{db_system="postgresql"}`) require.NoError(t, err) require.NotEmpty(t, results) }, test.Interval(time.Second)) diff --git a/test/oats/kafka/yaml/oats_python_kafka.yaml b/test/oats/kafka/yaml/oats_python_kafka.yaml index 7a2ad90fa..26d0fdb4d 100644 --- a/test/oats/kafka/yaml/oats_python_kafka.yaml +++ b/test/oats/kafka/yaml/oats_python_kafka.yaml @@ -11,6 +11,7 @@ expected: - traceql: '{ .messaging.operation.type = "process" }' spans: - name: 'my-topic process' + allow-duplicates: true attributes: messaging.destination.name: my-topic messaging.operation.type: process diff --git a/test/oats/sql/docker-compose-beyla-mysql.yml b/test/oats/sql/docker-compose-beyla-mysql.yml new file mode 100644 index 000000000..70296f642 --- /dev/null +++ b/test/oats/sql/docker-compose-beyla-mysql.yml @@ -0,0 +1,47 @@ +services: + # Use MySQL as a test SQL server + sqlserver: + build: + context: ../../integration/components/mysqldb + dockerfile: Dockerfile + image: mysql + ports: + - "3306:3306" + # Simple python HTTP server, which exposes one endpoint /query that does SQL query + testserver: + build: + context: ../../integration/components/pythonsql + dockerfile: Dockerfile_mysql + image: pysqlclient + ports: + - "8080:8080" + depends_on: + sqlserver: + condition: service_started + # eBPF auto instrumenter + autoinstrumenter: + build: + context: ../../.. + dockerfile: ./test/integration/components/beyla/Dockerfile + command: + - --config=/configs/instrumenter-config-traces-sql-text.yml + volumes: + - {{ .ConfigDir }}:/configs + - ./testoutput/run:/var/run/beyla + - ../../../testoutput:/coverage + privileged: true # in some environments (not GH Pull Requests) you can set it to false and then cap_add: [ SYS_ADMIN ] + network_mode: "service:testserver" + pid: "service:testserver" + environment: + GOCOVERDIR: "/coverage" + BEYLA_TRACE_PRINTER: "text" + BEYLA_OPEN_PORT: {{ .ApplicationPort }} + BEYLA_SERVICE_NAMESPACE: "integration-test" + BEYLA_METRICS_INTERVAL: "10ms" + BEYLA_BPF_BATCH_TIMEOUT: "10ms" + BEYLA_LOG_LEVEL: "DEBUG" + BEYLA_BPF_DEBUG: "true" + OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318" + depends_on: + testserver: + condition: service_started diff --git a/test/oats/sql/yaml/oats_sql_other_langs.yaml b/test/oats/sql/yaml/oats_sql_other_langs.yaml index ecb96cc02..e7f1b6501 100644 --- a/test/oats/sql/yaml/oats_sql_other_langs.yaml +++ b/test/oats/sql/yaml/oats_sql_other_langs.yaml @@ -8,20 +8,20 @@ input: interval: 500ms expected: traces: - - traceql: '{ .db.operation.name = "SELECT" && .db.system = "other_sql"}' + - traceql: '{ .db.operation.name = "SELECT" && .db.system = "postgresql"}' spans: - name: 'SELECT accounting.contacts' attributes: db.operation.name: SELECT db.collection.name: accounting.contacts - db.system: other_sql + db.system: postgresql db.query.text: "SELECT * from accounting.contacts WHERE id=1" metrics: - - promql: 'db_client_operation_duration_sum{db_system="other_sql"}' + - promql: 'db_client_operation_duration_sum{db_system="postgresql"}' value: "> 0" - - promql: 'db_client_operation_duration_bucket{le="0", db_system="other_sql"}' + - promql: 'db_client_operation_duration_bucket{le="0", db_system="postgresql"}' value: "== 0" - - promql: 'db_client_operation_duration_bucket{le="10", db_system="other_sql"}' + - promql: 'db_client_operation_duration_bucket{le="10", db_system="postgresql"}' value: "> 0" - - promql: 'db_client_operation_duration_count{db_system="other_sql"}' + - promql: 'db_client_operation_duration_count{db_system="postgresql"}' value: "> 0" diff --git a/test/oats/sql/yaml/oats_sql_other_langs_args.yaml b/test/oats/sql/yaml/oats_sql_other_langs_args.yaml index 99662f065..f34a0049b 100644 --- a/test/oats/sql/yaml/oats_sql_other_langs_args.yaml +++ b/test/oats/sql/yaml/oats_sql_other_langs_args.yaml @@ -8,19 +8,19 @@ input: interval: 500ms expected: traces: - - traceql: '{ .db.operation.name = "SELECT" && .db.system = "other_sql"}' + - traceql: '{ .db.operation.name = "SELECT" && .db.system = "postgresql"}' spans: - name: 'SELECT accounting.contacts' attributes: db.operation.name: SELECT db.collection.name: accounting.contacts - db.system: other_sql + db.system: postgresql metrics: - - promql: 'db_client_operation_duration_sum{db_system="other_sql"}' + - promql: 'db_client_operation_duration_sum{db_system="postgresql"}' value: "> 0" - - promql: 'db_client_operation_duration_bucket{le="0", db_system="other_sql"}' + - promql: 'db_client_operation_duration_bucket{le="0", db_system="postgresql"}' value: "== 0" - - promql: 'db_client_operation_duration_bucket{le="10", db_system="other_sql"}' + - promql: 'db_client_operation_duration_bucket{le="10", db_system="postgresql"}' value: "> 0" - - promql: 'db_client_operation_duration_count{db_system="other_sql"}' + - promql: 'db_client_operation_duration_count{db_system="postgresql"}' value: "> 0" diff --git a/test/oats/sql/yaml/oats_sql_other_langs_mysql.yaml b/test/oats/sql/yaml/oats_sql_other_langs_mysql.yaml new file mode 100644 index 000000000..6d8612269 --- /dev/null +++ b/test/oats/sql/yaml/oats_sql_other_langs_mysql.yaml @@ -0,0 +1,27 @@ +docker-compose: + generator: generic + files: + - ../docker-compose-beyla-mysql.yml +input: + - path: '/query' + +interval: 500ms +expected: + traces: + - traceql: '{ .db.operation.name = "SELECT" && .db.system = "mysql"}' + spans: + - name: 'SELECT actor' + attributes: + db.operation.name: SELECT + db.collection.name: actor + db.system: mysql + db.query.text: "SELECT * from actor WHERE actor_id=1" + metrics: + - promql: 'db_client_operation_duration_sum{db_system="mysql"}' + value: "> 0" + - promql: 'db_client_operation_duration_bucket{le="0", db_system="mysql"}' + value: "== 0" + - promql: 'db_client_operation_duration_bucket{le="10", db_system="mysql"}' + value: "> 0" + - promql: 'db_client_operation_duration_count{db_system="mysql"}' + value: "> 0" diff --git a/test/oats/sql/yaml/oats_sql_other_langs_prep.yaml b/test/oats/sql/yaml/oats_sql_other_langs_prep.yaml index 99662f065..f34a0049b 100644 --- a/test/oats/sql/yaml/oats_sql_other_langs_prep.yaml +++ b/test/oats/sql/yaml/oats_sql_other_langs_prep.yaml @@ -8,19 +8,19 @@ input: interval: 500ms expected: traces: - - traceql: '{ .db.operation.name = "SELECT" && .db.system = "other_sql"}' + - traceql: '{ .db.operation.name = "SELECT" && .db.system = "postgresql"}' spans: - name: 'SELECT accounting.contacts' attributes: db.operation.name: SELECT db.collection.name: accounting.contacts - db.system: other_sql + db.system: postgresql metrics: - - promql: 'db_client_operation_duration_sum{db_system="other_sql"}' + - promql: 'db_client_operation_duration_sum{db_system="postgresql"}' value: "> 0" - - promql: 'db_client_operation_duration_bucket{le="0", db_system="other_sql"}' + - promql: 'db_client_operation_duration_bucket{le="0", db_system="postgresql"}' value: "== 0" - - promql: 'db_client_operation_duration_bucket{le="10", db_system="other_sql"}' + - promql: 'db_client_operation_duration_bucket{le="10", db_system="postgresql"}' value: "> 0" - - promql: 'db_client_operation_duration_count{db_system="other_sql"}' + - promql: 'db_client_operation_duration_count{db_system="postgresql"}' value: "> 0" diff --git a/test/oats/sql/yaml/oats_sql_other_langs_prep_mysql.yaml b/test/oats/sql/yaml/oats_sql_other_langs_prep_mysql.yaml new file mode 100644 index 000000000..65da6339f --- /dev/null +++ b/test/oats/sql/yaml/oats_sql_other_langs_prep_mysql.yaml @@ -0,0 +1,26 @@ +docker-compose: + generator: generic + files: + - ../docker-compose-beyla-mysql.yml +input: + - path: '/argquery' + +interval: 500ms +expected: + traces: + - traceql: '{ .db.operation.name = "SELECT" && .db.system = "mysql"}' + spans: + - name: 'SELECT actor' + attributes: + db.operation.name: SELECT + db.collection.name: actor + db.system: mysql + metrics: + - promql: 'db_client_operation_duration_sum{db_system="mysql"}' + value: "> 0" + - promql: 'db_client_operation_duration_bucket{le="0", db_system="mysql"}' + value: "== 0" + - promql: 'db_client_operation_duration_bucket{le="10", db_system="mysql"}' + value: "> 0" + - promql: 'db_client_operation_duration_count{db_system="mysql"}' + value: "> 0"