Refactor Prometheus Metrics (#14)

config/serve.yaml

@@ -11,4 +11,5 @@ endpoints:
 
 logging:
   json: true
+  metrics: false
   level: debug

internal/cmd/serve.go

@@ -124,6 +124,10 @@ func init() {
 	flags.Int("shutdown-timeout", shutdownTimeout, "Timeout (in seconds) to wait for requests to finish")
 	_ = viper.BindPFlag("endpoints.timeouts.shutdown", flags.Lookup("shutdown-timeout"))
 
+	viper.SetDefault("logging.metrics", false)
+	flags.Bool("log-metrics", false, "Set whether to log metrics port requests")
+	_ = viper.BindPFlag("logging.metrics", flags.Lookup("log-metrics"))
+
 	rootCmd.AddCommand(serveCmd)
 }
 

internal/serve/metrics/main.go

@@ -27,8 +27,11 @@ type Service struct {
 func NewService() *Service {
 	router := gin.New()
 
-	router.Use(middleware.Logger())
-	router.Use(middleware.Prometheus())
+	if viper.GetBool("logging.metrics") {
+		router.Use(middleware.Logger())
+	}
+
+	router.Use(middleware.Prometheus("metrics"))
 	router.Use(gin.Recovery())
 
 	proxies := viper.GetStringSlice("endpoints.proxies")

internal/serve/middleware/logger.go

@@ -21,11 +21,6 @@ func Logger() gin.HandlerFunc {
 					http.StatusUnauthorized,
 					http.StatusNotFound,
 				),
-				// slogg.IgnorePath(
-				// 	"/alive",
-				// 	"/healthz",
-				// 	"/metrics",
-				// ),
 			},
 		},
 	)

internal/serve/middleware/metrics.go

@@ -26,7 +26,7 @@ func NewMetrics(namespace string) *Metrics {
 				Name:      "request_duration_seconds",
 				Help:      "The latency of the HTTP requests.",
 				//nolint:mnd // these are the building blocks for buckets
-				Buckets: prometheus.ExponentialBuckets(0.005, 2, 10),
+				Buckets: prometheus.ExponentialBuckets(0.0005, 2, 12),
 			},
 			[]string{"service", "handler", "method", "path", "status"},
 		),

internal/serve/middleware/prometheus.go

@@ -2,109 +2,96 @@ package middleware
 
 import (
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
-	PrometheusDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Name: "http_duration_seconds",
-		Help: "Duration of HTTP requests.",
-	}, []string{"path"})
-
-	PrometheusCounter = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "http_requests_total",
-		Help: "Count of HTTP requests.",
-	}, []string{"path", "status"})
+	summary = promauto.NewSummaryVec(prometheus.SummaryOpts{
+		Subsystem: "http",
+		Name:      "response_seconds",
+		Help:      "Duration of HTTP requests.",
+		//nolint:mnd // ignore
+		MaxAge: 15 * time.Second,
+		//nolint:mnd // ignore
+		Objectives: map[float64]float64{0.25: 0.01, 0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+	}, []string{"service"})
+
+	duration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Subsystem: "http",
+		Name:      "response_endpoints_seconds",
+		Help:      "Duration of HTTP requests.",
+		//nolint:mnd // ignore
+		Buckets: prometheus.ExponentialBucketsRange(0.00001, 2, 15),
+	}, []string{"service", "method", "path", "status"})
+
+	requests = promauto.NewCounterVec(prometheus.CounterOpts{
+		Subsystem: "http",
+		Name:      "request_total",
+		Help:      "Count of HTTP requests.",
+	}, []string{"service", "method", "path", "status"})
+
+	requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Subsystem: "http",
+		Name:      "request_size_bytes",
+		Help:      "Size of the HTTP requests.",
+		//nolint:mnd // ignore
+		Buckets: prometheus.ExponentialBuckets(64, 2, 10),
+	}, []string{"service", "method", "path", "status"})
+
+	responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Subsystem: "http",
+		Name:      "response_size_bytes",
+		Help:      "Size of the HTTP responses.",
+		//nolint:mnd // ignore
+		Buckets: prometheus.ExponentialBuckets(2, 2, 16),
+	}, []string{"service", "method", "path", "status"})
+
+	active = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: "http",
+		Name:      "request_open",
+		Help:      "Number of requests being actively handled.",
+	}, []string{"service"})
 )
 
-// PrometheusMiddleware provides instrumentation for the API calls made to a
-// connected service, counting both the number of requests being processed, the
-// number requested in total, and the time taken to process those requests.
-func Prometheus() gin.HandlerFunc {
+// Prometheus provides instrumentation for the API calls made to a connected
+// service, counting both the number of requests being processed, the number
+// requested in total, and the time taken to process those requests.
+func Prometheus(service string) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		var timer *prometheus.Timer
+		method := strings.ToUpper(c.Request.Method)
+		path := c.FullPath()
 
-		if c.FullPath() != "" {
-			timer = prometheus.NewTimer(
-				PrometheusDuration.WithLabelValues(c.FullPath()),
-			)
-		}
+		active.WithLabelValues(service).Inc()
+		defer active.WithLabelValues(service).Dec()
 
-		c.Next()
+		timer := time.Now()
+		defer func(c *gin.Context, t time.Time) {
+			taken := time.Since(t).Seconds()
 
-		if timer != nil {
-			timer.ObserveDuration()
-			PrometheusCounter.WithLabelValues(c.FullPath(), fmt.Sprintf("%d", c.Writer.Status())).Inc()
-		}
+			status := fmt.Sprintf("%d", c.Writer.Status())
+			if status == "0" {
+				status = "200"
+			}
+
+			responseBytes := float64(c.Writer.Size())
+
+			requestBytes := float64(c.Request.ContentLength)
+			if requestBytes < 0 {
+				requestBytes = 0
+			}
+
+			requests.WithLabelValues(service, method, path, status).Inc()
+			duration.WithLabelValues(service, method, path, status).Observe(taken)
+			summary.WithLabelValues(service).Observe(taken)
+			requestSize.WithLabelValues(service, method, path, status).Observe(requestBytes)
+			responseSize.WithLabelValues(service, method, path, status).Observe(responseBytes)
+		}(c, timer)
+
+		c.Next()
 	}
 }
-
-// Measure abstracts the HTTP handler implementation by only requesting a reporter, this
-// reporter will return the required data to be measured.
-// it accepts a next function that will be called as the wrapped logic before and after
-// measurement actions.
-// func (m Middleware) Measure(handlerID string, reporter Reporter, next func()) {
-// 	ctx := reporter.Context()
-//
-// 	// func (r *reporter) Method() string { return r.c.Request.Method }
-// 	//
-// 	// func (r *reporter) Context() context.Context { return r.c.Request.Context() }
-// 	//
-// 	// func (r *reporter) URLPath() string { return r.c.FullPath() }
-// 	//
-// 	// func (r *reporter) StatusCode() int { return r.c.Writer.Status() }
-// 	//
-// 	// func (r *reporter) BytesWritten() int64 { return int64(r.c.Writer.Size()) }
-//
-// 	// If there isn't predefined handler ID we
-// 	// set that ID as the URL path.
-// 	hid := handlerID
-// 	if handlerID == "" {
-// 		hid = reporter.URLPath()
-// 	}
-//
-// 	// Measure inflights if required.
-// 	if !m.cfg.DisableMeasureInflight {
-// 		props := metrics.HTTPProperties{
-// 			Service: m.cfg.Service,
-// 			ID:      hid,
-// 		}
-// 		m.cfg.Recorder.AddInflightRequests(ctx, props, 1)
-// 		defer m.cfg.Recorder.AddInflightRequests(ctx, props, -1)
-// 	}
-//
-// 	// Start the timer and when finishing measure the duration.
-// 	start := time.Now()
-// 	defer func() {
-// 		duration := time.Since(start)
-//
-// 		// If we need to group the status code, it uses the
-// 		// first number of the status code because is the least
-// 		// required identification way.
-// 		var code string
-// 		if m.cfg.GroupedStatus {
-// 			code = fmt.Sprintf("%dxx", reporter.StatusCode()/100)
-// 		} else {
-// 			code = strconv.Itoa(reporter.StatusCode())
-// 		}
-//
-// 		props := metrics.HTTPReqProperties{
-// 			Service: m.cfg.Service,
-// 			ID:      hid,
-// 			Method:  reporter.Method(),
-// 			Code:    code,
-// 		}
-// 		m.cfg.Recorder.ObserveHTTPRequestDuration(ctx, props, duration)
-//
-// 		// Measure size of response if required.
-// 		if !m.cfg.DisableMeasureSize {
-// 			m.cfg.Recorder.ObserveHTTPResponseSize(ctx, props, reporter.BytesWritten())
-// 		}
-// 	}()
-//
-// 	// Call the wrapped logic.
-// 	next()
-// }.

internal/serve/web/main.go

@@ -25,7 +25,7 @@ func NewService() *Service {
 	router := gin.New()
 
 	router.Use(middleware.Logger())
-	router.Use(middleware.Prometheus())
+	router.Use(middleware.Prometheus("web"))
 	router.Use(gin.Recovery())
 
 	proxies := viper.GetStringSlice("endpoints.proxies")

schemas/serve.json

@@ -171,6 +171,9 @@
         "level": {
           "$ref": "#/$defs/logging-level"
         },
+        "metrics": {
+          "$ref": "#/$defs/logging-metrics"
+        },
         "json": {
           "$ref": "#/$defs/logging-json"
         }
@@ -181,9 +184,15 @@
       "type": "string",
       "enum": ["debug", "info", "warning", "error"]
     },
+    "logging-metrics": {
+      "description": "Set whether or not to log requests to the metrics port",
+      "type": "boolean",
+      "default": false
+    },
     "logging-json": {
       "description": "Set whether or not to use JSON-based structured logging",
-      "type": "boolean"
+      "type": "boolean",
+      "default": false
     }
   },
   "type": "object",