add metrics

Signed-off-by: Michael Nairn <[email protected]>

config/metallb/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- github.com/metallb/metallb/config/native?ref=v0.13.7

config/observability/k8s_prometheus_patch.yaml

@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: k8s
+  namespace: monitoring
+spec:
+  remoteWrite:
+    - url: http://thanos-receive-router:19291/api/v1/receive
+      writeRelabelConfigs:
+        - action: replace
+          replacement: cluster1
+          targetLabel: cluster_id

config/observability/kustomization.yaml

@@ -0,0 +1,16 @@
+resources:
+  - github.com/kuadrant/kuadrant-operator/config/observability?ref=main
+  - ./thanos
+  - github.com/kuadrant/kuadrant-operator/examples/dashboards?ref=main
+  - github.com/kuadrant/kuadrant-operator/examples/alerts?ref=main
+
+patches:
+  - target:
+      kind: ServiceMonitor
+    patch: |
+      $patch: delete
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: ANY
+  - path: k8s_prometheus_patch.yaml

config/observability/thanos/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+  - github.com/kuadrant/kuadrant-operator/config/thanos?ref=main
+
+patches:
+  - patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: monitoring

test/scale/README.md

@@ -1,5 +1,28 @@
 
+Create a kind cluster with prometheus/thanos installed and configured
 ```shell
-make kube-burner
-./bin/kube-burner init  -c config.yaml
-```
+make local-setup
+kubectl apply --server-side -k config/observability
+kubectl apply --server-side -k config/observability # Run twice if it fails the first time
+```
+
+Forward port for prometheus
+```shell
+kubectl -n monitoring port-forward service/thanos-query 9090:9090
+```
+
+Forward port for graphana (Optional)
+```shell
+kubectl -n monitoring port-forward service/grafana 3000:3000
+```
+Access dashboards http://127.0.0.1:3000
+
+Tail all operator logs (Optional)
+```shell
+kubectl stern -l control-plane=dns-operator-controller-manager -A
+```
+
+Run default scale test(1 iteration using the inmemory provider)
+```shell
+PROMETHEUS_URL=http://127.0.0.1:9090 PROMETHEUS_TOKEN="" make test-scale
+```

test/scale/config.yaml

@@ -1,3 +1,12 @@
+metricsEndpoints:
+  - endpoint: {{ .PROMETHEUS_URL }}
+    token: {{ .PROMETHEUS_TOKEN }}
+    metrics:
+      - ./metrics.yaml
+    indexer:
+      type: local
+      metricsDirectory: ./metrics
+
 jobs:
   - name: scale-test-setup-dns-operator-cluster-roles
     jobIterations: 1
@@ -35,6 +44,12 @@ jobs:
       - objectTemplate: ./dns-operator/dns-operator-deployment.yaml
         kind: Deployment
         replicas: 1
+      - objectTemplate: ./dns-operator/metrics-service.yaml
+        kind: Service
+        replicas: 1
+      - objectTemplate: ./dns-operator/service-monitor.yaml
+        kind: ServiceMonitor
+        replicas: 1
   - name: scale-test-setup
     jobType: create #default
     jobIterations: {{ .JOB_ITERATIONS }}

test/scale/dns-operator/dns-operator-deployment.yaml

@@ -19,6 +19,7 @@ spec:
       containers:
       - args:
         - --leader-elect
+        - --metrics-bind-address=:8080
         - --provider=aws,google,inmemory,azure
         - --zap-log-level=debug
         command:
@@ -34,6 +35,9 @@ spec:
           initialDelaySeconds: 15
           periodSeconds: 20
         name: manager
+        ports:
+          - containerPort: 8080
+            name: metrics
         readinessProbe:
           httpGet:
             path: /readyz

test/scale/dns-operator/kustomization.yaml

@@ -1,5 +1,5 @@
 resources:
-- ../../../config/manager
+- ../../../config/default
 
 patches:
   - patch: |-

test/scale/dns-operator/metrics-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: dns-operator-controller-manager
+  name: dns-operator-controller-manager-metrics-service
+spec:
+  ports:
+  - name: metrics
+    port: 8080
+    targetPort: metrics
+  selector:
+    control-plane: dns-operator-controller-manager

test/scale/dns-operator/service-monitor.yaml

@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: dns-operator-metrics-monitor
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+      scheme: http
+  selector:
+    matchLabels:
+      control-plane: dns-operator-controller-manager

test/scale/metrics.yaml

@@ -0,0 +1,2 @@
+- query: sum(rate(container_cpu_usage_seconds_total{container="",namespace=~"kuadrant-system|kuadrant-dns-operator-*|scale-test-.*"}[5m])) by(namespace)
+  metricName: namespaceCPU

test/scale/metrics/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore