From 30ed77a40f1e9bca90ace9ea887d0bf19f4a98ae Mon Sep 17 00:00:00 2001 From: Maskym Vavilov Date: Mon, 25 Nov 2024 14:32:22 +0000 Subject: [PATCH] probes unhealthy only if threshold exeeded Signed-off-by: Maskym Vavilov --- internal/controller/dnsrecord_healthchecks.go | 16 +++++++--------- .../controller/dnsrecord_healthchecks_test.go | 3 --- internal/probes/worker.go | 5 ++++- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/internal/controller/dnsrecord_healthchecks.go b/internal/controller/dnsrecord_healthchecks.go index d97f98e..38a4a67 100644 --- a/internal/controller/dnsrecord_healthchecks.go +++ b/internal/controller/dnsrecord_healthchecks.go @@ -136,20 +136,18 @@ func removeUnhealthyEndpoints(specEndpoints []*endpoint.Endpoint, dnsRecord *v1a var haveHealthyProbes bool for _, probe := range probes.Items { - // if the probe is healthy or unknown, continue to the next probe + // if the probe is healthy, continue to the next probe if probe.Status.Healthy != nil && *probe.Status.Healthy { haveHealthyProbes = true continue } - // if we exceeded a threshold or we haven't probed yet - if probe.Status.ConsecutiveFailures >= dnsRecord.Spec.HealthCheck.FailureThreshold || probe.Status.Healthy == nil { - //delete bad endpoint from all endpoints targets - tree.RemoveNode(&common.DNSTreeNode{ - Name: probe.Spec.Address, - }) - unhealthyAddresses = append(unhealthyAddresses, probe.Spec.Address) - } + // if unhealthy or we haven't probed yet + //delete bad endpoint from all endpoints targets + tree.RemoveNode(&common.DNSTreeNode{ + Name: probe.Spec.Address, + }) + unhealthyAddresses = append(unhealthyAddresses, probe.Spec.Address) } diff --git a/internal/controller/dnsrecord_healthchecks_test.go b/internal/controller/dnsrecord_healthchecks_test.go index a182588..6a16e62 100644 --- a/internal/controller/dnsrecord_healthchecks_test.go +++ b/internal/controller/dnsrecord_healthchecks_test.go @@ -453,7 +453,6 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() { if probe.Spec.Address == "172.32.200.1" { probe.Status.Healthy = ptr.To(false) probe.Status.LastCheckedAt = metav1.Now() - probe.Status.ConsecutiveFailures = dnsRecord.Spec.HealthCheck.FailureThreshold + 1 g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed()) updated = true } @@ -499,7 +498,6 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() { if probe.Spec.Address == "172.32.200.2" { probe.Status.Healthy = ptr.To(false) probe.Status.LastCheckedAt = metav1.Now() - probe.Status.ConsecutiveFailures = dnsRecord.Spec.HealthCheck.FailureThreshold + 1 g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed()) updated = true } @@ -585,7 +583,6 @@ var _ = Describe("DNSRecordReconciler_HealthChecks", func() { if probe.Spec.Address == "172.32.200.1" { probe.Status.Healthy = ptr.To(false) probe.Status.LastCheckedAt = metav1.Now() - probe.Status.ConsecutiveFailures = dnsRecord.Spec.HealthCheck.FailureThreshold + 1 g.Expect(k8sClient.Status().Update(ctx, &probe)).To(Succeed()) updated = true } diff --git a/internal/probes/worker.go b/internal/probes/worker.go index c723c47..de9da6f 100644 --- a/internal/probes/worker.go +++ b/internal/probes/worker.go @@ -257,11 +257,14 @@ func (w *Probe) Start(clientctx context.Context, k8sClient client.Client, probe freshProbe.Status.ObservedGeneration = freshProbe.Generation if !probeResult.Healthy { freshProbe.Status.ConsecutiveFailures++ + if freshProbe.Status.ConsecutiveFailures > freshProbe.Spec.FailureThreshold { + freshProbe.Status.Healthy = &probeResult.Healthy + } } else { freshProbe.Status.ConsecutiveFailures = 0 + freshProbe.Status.Healthy = &probeResult.Healthy } logger.V(1).Info("health: execution complete ", "result", probeResult, "checked at", probeResult.CheckedAt.String(), "previoud check at ", probeResult.PreviousCheck) - freshProbe.Status.Healthy = &probeResult.Healthy freshProbe.Status.LastCheckedAt = probeResult.CheckedAt freshProbe.Status.Reason = probeResult.Reason freshProbe.Status.Status = probeResult.Status