tests: Add kube burner scale test

Signed-off-by: Michael Nairn <[email protected]>

Makefile

@@ -175,6 +175,17 @@ test-e2e: ginkgo
 test-e2e-multi: ginkgo
 	$(GINKGO) $(GINKGO_FLAGS) -tags=e2e --label-filter=multi_record ./test/e2e
 
+.PHONY: test-scale
+test-scale: export JOB_ITERATIONS := 1
+test-scale: export KUADRANT_ZONE_ROOT_DOMAIN := kuadrant.local
+test-scale: export DNS_PROVIDER := inmemory
+test-scale: export PROMETHEUS_URL := http://127.0.0.1:9090
+test-scale: export PROMETHEUS_TOKEN := ""
+test-scale: export SKIP_CLEANUP := false
+test-scale: kube-burner
+	@echo "test-scale: JOB_ITERATIONS=${JOB_ITERATIONS} KUADRANT_ZONE_ROOT_DOMAIN=${KUADRANT_ZONE_ROOT_DOMAIN} DNS_PROVIDER=${DNS_PROVIDER} PROMETHEUS_URL=${PROMETHEUS_URL} PROMETHEUS_TOKEN=${PROMETHEUS_TOKEN}"
+	cd test/scale && $(KUBE_BURNER) init -c config.yaml --log-level debug
+
 .PHONY: local-setup-cluster
 local-setup-cluster: DEPLOY=false
 local-setup-cluster: TEST_NAMESPACE=dnstest
@@ -338,6 +349,7 @@ YQ = $(LOCALBIN)/yq
 GINKGO ?= $(LOCALBIN)/ginkgo
 GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
 HELM ?= $(LOCALBIN)/helm
+KUBE_BURNER ?= $(LOCALBIN)/kube-burner
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.5.0
@@ -349,6 +361,7 @@ YQ_VERSION := v4.34.2
 GINKGO_VERSION ?= v2.17.1
 GOLANGCI_LINT_VERSION ?= v1.55.2
 HELM_VERSION = v3.15.0
+KUBE_BURNER_VERSION = v1.11.1
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading.
@@ -431,6 +444,20 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(LOCALBIN) $(GOLANGCI_LINT_VERSION)
 
+.PHONY: kube-burner
+kube-burner: $(KUBE_BURNER) ## Download kube-burner locally if necessary.
+$(KUBE_BURNER):
+	@{ \
+	set -e ;\
+	mkdir -p $(dir $(KUBE_BURNER)) ;\
+	OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
+	wget -O kube-burner.tar.gz https://github.com/kube-burner/kube-burner/releases/download/v1.11.1/kube-burner-V1.11.1-linux-x86_64.tar.gz ;\
+	tar -zxvf kube-burner.tar.gz ;\
+	mv kube-burner $(KUBE_BURNER) ;\
+	chmod +x $(KUBE_BURNER) ;\
+	rm -rf $${OS}-$${ARCH} kube-burner.tar.gz ;\
+	}
+
 .PHONY: bundle
 bundle: manifests manifests-gen-base-csv kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files.
 	$(OPERATOR_SDK) generate kustomize manifests -q

config/metallb/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- github.com/metallb/metallb/config/native?ref=v0.13.7

config/observability/k8s_prometheus_patch.yaml

@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: k8s
+  namespace: monitoring
+spec:
+  remoteWrite:
+    - url: http://thanos-receive-router:19291/api/v1/receive
+      writeRelabelConfigs:
+        - action: replace
+          replacement: cluster1
+          targetLabel: cluster_id

config/observability/kustomization.yaml

@@ -0,0 +1,37 @@
+resources:
+  - ./metrics-server
+  - github.com/kuadrant/kuadrant-operator/config/observability?ref=main
+  - ./thanos
+  - github.com/kuadrant/kuadrant-operator/examples/dashboards?ref=main
+  - github.com/kuadrant/kuadrant-operator/examples/alerts?ref=main
+
+patches:
+  - patch: |
+      $patch: delete
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: authorino-operator-metrics
+        namespace: kuadrant-system
+  - patch: |
+      $patch: delete
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: dns-operator-metrics-monitor
+        namespace: kuadrant-system
+  - patch: |
+      $patch: delete
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: kuadrant-operator-metrics
+        namespace: kuadrant-system
+  - patch: |
+      $patch: delete
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: limitador-operator-metrics
+        namespace: kuadrant-system
+  - path: k8s_prometheus_patch.yaml

config/observability/metrics-server/kustomization.yaml

@@ -0,0 +1,24 @@
+resources:
+  - https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/components.yaml
+patches:
+  - patch: |-
+      - op: add
+        path: /spec/template/spec/containers/0/args/-
+        value: --kubelet-insecure-tls
+    target:
+      version: v1
+      kind: Deployment
+      name: metrics-server
+      namespace: kube-system
+  - patch: |
+      $patch: delete
+      apiVersion: apiregistration.k8s.io/v1
+      kind: APIService
+      metadata:
+        name: v1beta1.metrics.k8s.io
+  - patch: |
+      $patch: delete
+      apiVersion: rbac.authorization.k8s.io/v1
+      kind: ClusterRole
+      metadata:
+        name: system:aggregated-metrics-reader

config/observability/thanos/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+  - github.com/kuadrant/kuadrant-operator/config/thanos?ref=main
+
+patches:
+  - patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: monitoring

test/scale/README.md

@@ -0,0 +1,77 @@
+# DNS Operator Scale Test
+
+Scale testing using [kube-burner](https://kube-burner.github.io/kube-burner/latest).
+
+
+## Setup local environment (kind)
+
+Create a kind cluster with prometheus/thanos installed and configured
+```shell
+make local-setup
+kubectl apply --server-side -k config/observability
+kubectl apply --server-side -k config/observability # Run twice if it fails the first time
+```
+
+Forward port for prometheus
+```shell
+kubectl -n monitoring port-forward service/thanos-query 9090:9090
+```
+
+Forward port for grafana (Optional)
+```shell
+kubectl -n monitoring port-forward service/grafana 3000:3000
+```
+Access dashboards http://127.0.0.1:3000
+
+Tail all operator logs (Optional)
+```shell
+kubectl stern -l control-plane=dns-operator-controller-manager -A
+```
+
+## Run scale test
+
+Export Environment variables:
+```shell
+#All
+export SKIP_CLEANUP=false
+export PROMETHEUS_URL=http://127.0.0.1:9090 
+export PROMETHEUS_TOKEN=""
+#AWS
+export KUADRANT_AWS_ACCESS_KEY_ID=<my aws access key id>
+export KUADRANT_AWS_SECRET_ACCESS_KEY=<my aws secret access key>
+export KUADRANT_AWS_REGION=""
+#GCP
+export KUADRANT_GCP_GOOGLE_CREDENTIALS=<my gcp credentals json>
+export KUADRANT_GCP_PROJECT_ID=<my gcp project id>
+#Azure
+export KUADRANT_AZURE_CREDENTIALS=<my azure credentials json>
+```
+
+### inmemory
+
+```shell
+make test-scale JOB_ITERATIONS=2
+```
+### aws
+
+```shell
+make test-scale JOB_ITERATIONS=2 DNS_PROVIDER=aws KUADRANT_ZONE_ROOT_DOMAIN=<my aws hosted domain>
+```
+
+### gcp
+
+```shell
+make test-scale JOB_ITERATIONS=2 DNS_PROVIDER=gcp KUADRANT_ZONE_ROOT_DOMAIN=<my gcp hosted domain>
+```
+
+### azure
+
+```shell
+make test-scale JOB_ITERATIONS=2 DNS_PROVIDER=azure KUADRANT_ZONE_ROOT_DOMAIN=<my azure hosted domain>
+```
+
+## Checking alerts
+
+```shell
+./bin/kube-burner check-alerts -u $PROMETHEUS_URL -t '$PROMETHEUS_TOKEN' -a test/scale/alerts.yaml
+```

test/scale/alerts.yaml

@@ -0,0 +1,3 @@
+- expr: increase(kube_pod_container_status_restarts_total{container="manager", namespace=~"kuadrant-system|kuadrant-dns-operator-.*"}[5m]) > 0
+  description: manager pod restarts
+  severity: error

test/scale/aws-credentials.yaml

@@ -0,0 +1,9 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: aws-credentials
+stringData:
+  AWS_ACCESS_KEY_ID: '{{ .KUADRANT_AWS_ACCESS_KEY_ID }}'
+  AWS_REGION: '{{ .KUADRANT_AWS_REGION }}'
+  AWS_SECRET_ACCESS_KEY: '{{ .KUADRANT_AWS_SECRET_ACCESS_KEY }}'
+type: kuadrant.io/aws

test/scale/azure-credentials.yaml

@@ -0,0 +1,7 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: azure-credentials
+stringData:
+  azure.json: '{{ .KUADRANT_AZURE_CREDENTIALS }}'
+type: kuadrant.io/azure

test/scale/config.yaml

@@ -0,0 +1,135 @@
+metricsEndpoints:
+  - endpoint: {{ .PROMETHEUS_URL }}
+    token: {{ .PROMETHEUS_TOKEN }}
+    metrics:
+      - ./metrics.yaml
+    alerts:
+      - ./alerts.yaml
+    indexer:
+      type: local
+      metricsDirectory: ./metrics
+
+jobs:
+  - name: scale-test-setup-dns-operator-cluster-roles
+    jobIterations: 1
+    verifyObjects: false
+    objects:
+      - objectTemplate: ../../config/rbac/role.yaml
+        kind: ClusterRole
+        runOnce: true
+        replicas: 1
+  - name: scale-test-setup-dns-operator
+    jobType: create #default
+    jobIterations: {{ .JOB_ITERATIONS }}
+    namespacedIterations: true #default
+    namespace: kuadrant-dns-operator
+    waitWhenFinished: true #default
+    maxWaitTimeout: 5m
+    preLoadImages: false
+    objects:
+      - objectTemplate: ../../config/rbac/leader_election_role.yaml
+        kind: Role
+        replicas: 1
+      - objectTemplate: ./dns-operator/role-binding.yaml
+        kind: ClusterRoleBinding
+        replicas: 1
+        inputVars:
+          Namespace: "kuadrant-dns-operator"
+      - objectTemplate: ./dns-operator/leader-election-role-binding.yaml
+        kind: RoleBinding
+        replicas: 1
+        inputVars:
+          Namespace: "kuadrant-dns-operator"
+      - objectTemplate: ./dns-operator/service-account.yaml
+        kind: ServiceAccount
+        replicas: 1
+      - objectTemplate: ./dns-operator/dns-operator-deployment.yaml
+        kind: Deployment
+        replicas: 1
+      - objectTemplate: ./dns-operator/metrics-service.yaml
+        kind: Service
+        replicas: 1
+      - objectTemplate: ./dns-operator/service-monitor.yaml
+        kind: ServiceMonitor
+        replicas: 1
+  - name: scale-test-setup
+    jobType: create #default
+    jobIterations: {{ .JOB_ITERATIONS }}
+    namespacedIterations: true #default
+    namespace: scale-test
+    waitWhenFinished: true #default
+    maxWaitTimeout: 5m
+    preLoadImages: false
+    objects:
+      - objectTemplate: ./httpbin/deployment.yaml
+        kind: Deployment
+        replicas: 1
+      - objectTemplate: ./httpbin/service.yaml
+        kind: Service
+        replicas: 1
+      {{if (eq .DNS_PROVIDER "inmemory")}}
+      - objectTemplate: ./inmemory-credentials.yaml
+        kind: Secret
+        replicas: 1
+        inputVars:
+          KUADRANT_ZONE_ROOT_DOMAIN: '{{ .KUADRANT_ZONE_ROOT_DOMAIN }}'
+      {{end}}
+      {{if (eq .DNS_PROVIDER "aws")}}
+      - objectTemplate: ./aws-credentials.yaml
+        kind: Secret
+        replicas: 1
+        inputVars:
+          KUADRANT_AWS_ACCESS_KEY_ID: '{{ .KUADRANT_AWS_ACCESS_KEY_ID }}'
+          KUADRANT_AWS_REGION: '{{ .KUADRANT_AWS_REGION }}'
+          KUADRANT_AWS_SECRET_ACCESS_KEY: '{{ .KUADRANT_AWS_SECRET_ACCESS_KEY }}'
+      {{end}}
+      {{if (eq .DNS_PROVIDER "gcp")}}
+      - objectTemplate: ./gcp-credentials.yaml
+        kind: Secret
+        replicas: 1
+        inputVars:
+          KUADRANT_GCP_GOOGLE_CREDENTIALS: '{{ .KUADRANT_GCP_GOOGLE_CREDENTIALS }}'
+          KUADRANT_GCP_PROJECT_ID: '{{ .KUADRANT_GCP_PROJECT_ID }}'
+      {{end}}
+      {{if (eq .DNS_PROVIDER "azure")}}
+      - objectTemplate: ./azure-credentials.yaml
+        kind: Secret
+        replicas: 1
+        inputVars:
+          KUADRANT_AZURE_CREDENTIALS: '{{ .KUADRANT_AZURE_CREDENTIALS }}'
+      {{end}}
+  - name: scale-test-loadbalanced
+    jobType: create #default
+    jobIterations: {{ .JOB_ITERATIONS }}
+    namespacedIterations: true #default
+    namespace: scale-test
+    waitWhenFinished: true #default
+    maxWaitTimeout: 2m
+    objects:
+      - objectTemplate: ./dnsrecord-loadbalanced.yaml
+        kind: DNSRecord
+        replicas: 1
+        inputVars:
+          KUADRANT_ZONE_ROOT_DOMAIN: {{ .KUADRANT_ZONE_ROOT_DOMAIN }}
+          DNS_PROVIDER: {{ .DNS_PROVIDER }}
+          GeoCode: {{if (eq .DNS_PROVIDER "aws")}} GEO-EU {{else if (eq .DNS_PROVIDER "gcp")}} europe-west1 {{else if (eq .DNS_PROVIDER "azure")}} GEO-EU {{else}} foo {{end}}
+        waitOptions:
+          forCondition: "Ready"
+  - name: scale-test-dnsrecord-read
+    jobType: read
+    jobIterations: 1
+    jobPause: 30s
+    objects:
+      - kind: DNSRecord
+        apiVersion: kuadrant.io/v1alpha1
+        labelSelector: {kube-burner-job: scale-test-loadbalanced}
+  {{ if (eq .SKIP_CLEANUP "true") }}
+    # nothing to do if cleanup is skipped
+  {{ else }}
+  - name: scale-test-dnsrecord-cleanup
+    jobType: delete
+    objects:
+      - kind: DNSRecord
+        apiVersion: kuadrant.io/v1alpha1
+        labelSelector: {kube-burner-job: scale-test-loadbalanced}
+  {{ end }}