Merge pull request #315 from Kuadrant/gh-294-2

probes unhealthy only if threshold exeeded
Kuadrant · Nov 25, 2024 · bee684b · bee684b
2 parents bf2d33b + 30ed77a
commit bee684b
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 13 deletions.
diff --git a/internal/controller/dnsrecord_healthchecks.go b/internal/controller/dnsrecord_healthchecks.go
@@ -136,20 +136,18 @@ func removeUnhealthyEndpoints(specEndpoints []*endpoint.Endpoint, dnsRecord *v1a
 
 	var haveHealthyProbes bool
 	for _, probe := range probes.Items {
-		// if the probe is healthy or unknown, continue to the next probe
+		// if the probe is healthy, continue to the next probe
 		if probe.Status.Healthy != nil && *probe.Status.Healthy {
 			haveHealthyProbes = true
 			continue
 		}
 
-		// if we exceeded a threshold or we haven't probed yet
-		if probe.Status.ConsecutiveFailures >= dnsRecord.Spec.HealthCheck.FailureThreshold || probe.Status.Healthy == nil {
-			//delete bad endpoint from all endpoints targets
-			tree.RemoveNode(&common.DNSTreeNode{
-				Name: probe.Spec.Address,
-			})
-			unhealthyAddresses = append(unhealthyAddresses, probe.Spec.Address)
-		}
+		// if unhealthy or we haven't probed yet
+		//delete bad endpoint from all endpoints targets
+		tree.RemoveNode(&common.DNSTreeNode{
+			Name: probe.Spec.Address,
+		})
+		unhealthyAddresses = append(unhealthyAddresses, probe.Spec.Address)
 
 	}
 

diff --git a/internal/controller/dnsrecord_healthchecks_test.go b/internal/controller/dnsrecord_healthchecks_test.go
@@ -453,7 +453,6 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() {
 				if probe.Spec.Address == "172.32.200.1" {
 					probe.Status.Healthy = ptr.To(false)
 					probe.Status.LastCheckedAt = metav1.Now()
-					probe.Status.ConsecutiveFailures = dnsRecord.Spec.HealthCheck.FailureThreshold + 1
 					g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed())
 					updated = true
 				}
@@ -499,7 +498,6 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() {
 				if probe.Spec.Address == "172.32.200.2" {
 					probe.Status.Healthy = ptr.To(false)
 					probe.Status.LastCheckedAt = metav1.Now()
-					probe.Status.ConsecutiveFailures = dnsRecord.Spec.HealthCheck.FailureThreshold + 1
 					g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed())
 					updated = true
 				}
@@ -585,7 +583,6 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() {
 				if probe.Spec.Address == "172.32.200.1" {
 					probe.Status.Healthy = ptr.To(false)
 					probe.Status.LastCheckedAt = metav1.Now()
-					probe.Status.ConsecutiveFailures = dnsRecord.Spec.HealthCheck.FailureThreshold + 1
 					g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed())
 					updated = true
 				}

diff --git a/internal/probes/worker.go b/internal/probes/worker.go
@@ -257,11 +257,14 @@ func (w *Probe) Start(clientctx context.Context, k8sClient client.Client, probe
 			freshProbe.Status.ObservedGeneration = freshProbe.Generation
 			if !probeResult.Healthy {
 				freshProbe.Status.ConsecutiveFailures++
+				if freshProbe.Status.ConsecutiveFailures > freshProbe.Spec.FailureThreshold {
+					freshProbe.Status.Healthy = &probeResult.Healthy
+				}
 			} else {
 				freshProbe.Status.ConsecutiveFailures = 0
+				freshProbe.Status.Healthy = &probeResult.Healthy
 			}
 			logger.V(1).Info("health: execution complete ", "result", probeResult, "checked at", probeResult.CheckedAt.String(), "previoud check at ", probeResult.PreviousCheck)
-			freshProbe.Status.Healthy = &probeResult.Healthy
 			freshProbe.Status.LastCheckedAt = probeResult.CheckedAt
 			freshProbe.Status.Reason = probeResult.Reason
 			freshProbe.Status.Status = probeResult.Status