[SECURESIGN-1044] Error handler

bundle/manifests/rhtas-operator.clusterserviceversion.yaml

@@ -192,7 +192,7 @@ metadata:
       ]
     capabilities: Seamless Upgrades
     containerImage: registry.redhat.io/rhtas/rhtas-rhel9-operator@sha256:a21f7128694a64989bf0d84a7a7da4c1ffc89edf62d594dc8bea7bcfe9ac08d3
-    createdAt: "2024-06-12T14:09:20Z"
+    createdAt: "2024-06-14T12:17:18Z"
     features.operators.openshift.io/cnf: "false"
     features.operators.openshift.io/cni: "false"
     features.operators.openshift.io/csi: "false"

internal/controller/common/action/base_action.go

@@ -70,8 +70,9 @@ func (action *BaseAction) FailedWithStatusUpdate(ctx context.Context, err error,
 		}
 		err = errors.Join(e, err)
 	}
-	// Requeue will be caused by update
-	return &Result{Result: reconcile.Result{Requeue: false}, Err: err}
+	// Requeue is disabled for Failed objects
+	// wait for 30 second and invoke error-handler
+	return &Result{Result: reconcile.Result{RequeueAfter: 30 * time.Second}}
 }
 
 func (action *BaseAction) Return() *Result {

internal/controller/constants/config.go

@@ -1,5 +1,5 @@
 package constants
 
 var (
-	CreateTreeDeadline int64 = 1200
+	CreateTreeDeadline int64 = 30
 )

internal/controller/rekor/actions/error.go

@@ -0,0 +1,62 @@
+package actions
+
+import (
+	"context"
+
+	rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1"
+	"github.com/securesign/operator/internal/controller/common/action"
+	"github.com/securesign/operator/internal/controller/constants"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func NewHandleErrorAction() action.Action[rhtasv1alpha1.Rekor] {
+	return &handleErrorAction{}
+}
+
+type handleErrorAction struct {
+	action.BaseAction
+}
+
+func (i handleErrorAction) Name() string {
+	return "error handler"
+}
+
+func (i handleErrorAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
+	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
+	return c.Reason == constants.Failure
+}
+
+func (i handleErrorAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result {
+	i.Recorder.Event(instance, v1.EventTypeWarning, constants.Failure, "Restarted by error handler")
+
+	newStatus := rhtasv1alpha1.RekorStatus{}
+
+	// - keep the status.treeId if not nil
+	newStatus.TreeID = instance.Status.TreeID
+	// - keep the status.pvcName if not nil
+	newStatus.PvcName = instance.Status.PvcName
+
+	if meta.IsStatusConditionTrue(instance.Status.Conditions, SignerCondition) {
+		instance.Status.Signer.DeepCopyInto(&newStatus.Signer)
+		newStatus.Conditions = append(newStatus.Conditions, *meta.FindStatusCondition(instance.Status.Conditions, SignerCondition))
+	}
+
+	if meta.IsStatusConditionTrue(instance.Status.Conditions, ServerCondition) {
+		instance.Status.ServerConfigRef.DeepCopyInto(newStatus.ServerConfigRef)
+		// do not append server condition - let controller to redeploy
+	}
+
+	meta.SetStatusCondition(&newStatus.Conditions, metav1.Condition{
+		Type:    constants.Ready,
+		Status:  metav1.ConditionFalse,
+		Reason:  constants.Pending,
+		Message: "Restarted by error handler",
+	})
+	instance.Status = newStatus
+	return i.StatusUpdate(ctx, instance)
+}

internal/controller/rekor/actions/pending.go

@@ -26,6 +26,9 @@ func (i pendingAction) Name() string {
 
 func (i pendingAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Pending
 }
 

internal/controller/rekor/actions/rbac.go

@@ -29,6 +29,9 @@ func (i rbacAction) Name() string {
 
 func (i rbacAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating || c.Reason == constants.Ready
 }
 

internal/controller/rekor/actions/server/createTree.go

@@ -30,6 +30,9 @@ func (i createTrillianTreeAction) Name() string {
 
 func (i createTrillianTreeAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating && instance.Status.TreeID == nil
 }
 

internal/controller/rekor/actions/server/deployment.go

@@ -29,6 +29,9 @@ func (i deployAction) Name() string {
 
 func (i deployAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating || c.Reason == constants.Ready
 }
 

internal/controller/rekor/actions/server/generate_signer.go

@@ -43,6 +43,9 @@ func (g generateSigner) Name() string {
 
 func (g generateSigner) CanHandle(_ context.Context, instance *v1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	if c.Reason != constants.Pending && c.Reason != constants.Ready {
 		return false
 	}
@@ -165,12 +168,6 @@ func (g generateSigner) Handle(ctx context.Context, instance *v1alpha1.Rekor) *a
 	} else {
 		instance.Status.Signer.PasswordRef = instance.Spec.Signer.PasswordRef
 	}
-	meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
-		Type:    constants.Ready,
-		Status:  metav1.ConditionFalse,
-		Reason:  constants.Creating,
-		Message: "Signer resolved",
-	})
 	meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
 		Type:   actions.ServerCondition,
 		Status: metav1.ConditionFalse,

internal/controller/rekor/actions/server/ingress.go

@@ -30,6 +30,9 @@ func (i ingressAction) Name() string {
 
 func (i ingressAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return (c.Reason == constants.Creating || c.Reason == constants.Ready) &&
 		instance.Spec.ExternalAccess.Enabled
 }

internal/controller/rekor/actions/server/initialize.go

@@ -26,6 +26,9 @@ func (i initializeAction) Name() string {
 
 func (i initializeAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Initialize && !meta.IsStatusConditionTrue(instance.Status.Conditions, actions.ServerCondition)
 }
 

internal/controller/rekor/actions/server/monitoring.go

@@ -30,6 +30,9 @@ func (i monitoringAction) Name() string {
 
 func (i monitoringAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return (c.Reason == constants.Creating || c.Reason == constants.Ready) && instance.Spec.Monitoring.Enabled
 }
 

internal/controller/rekor/actions/server/pvc.go

@@ -34,6 +34,9 @@ func (i createPvcAction) Name() string {
 
 func (i createPvcAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating && instance.Status.PvcName == ""
 }
 

internal/controller/rekor/actions/server/resolve_pub_key.go

@@ -37,6 +37,9 @@ func (i resolvePubKeyAction) Name() string {
 
 func (i resolvePubKeyAction) CanHandle(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	if (c.Reason != constants.Initialize && c.Reason != constants.Ready) || !meta.IsStatusConditionTrue(instance.Status.Conditions, actions.ServerCondition) {
 		return false
 	}

internal/controller/rekor/actions/server/servrConfig.go

@@ -32,6 +32,9 @@ func (i serverConfig) Name() string {
 
 func (i serverConfig) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating && instance.Status.ServerConfigRef == nil
 }
 

internal/controller/rekor/actions/server/status_url.go

@@ -28,6 +28,9 @@ func (i statusUrlAction) Name() string {
 
 func (i statusUrlAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating || c.Reason == constants.Ready
 }
 

internal/controller/rekor/actions/server/svc.go

@@ -31,6 +31,9 @@ func (i createServiceAction) Name() string {
 
 func (i createServiceAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating || c.Reason == constants.Ready
 }
 

internal/controller/rekor/actions/transitions/to_create_phase.go

@@ -0,0 +1,39 @@
+package transitions
+
+import (
+	"context"
+
+	rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1"
+	"github.com/securesign/operator/internal/controller/common/action"
+	"github.com/securesign/operator/internal/controller/constants"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func NewToCreateAction() action.Action[rhtasv1alpha1.Rekor] {
+	return &toCreateAction{}
+}
+
+type toCreateAction struct {
+	action.BaseAction
+}
+
+func (i toCreateAction) Name() string {
+	return "move to create phase"
+}
+
+func (i toCreateAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
+	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
+	return c.Reason == constants.Pending
+}
+
+func (i toCreateAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result {
+
+	meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{Type: constants.Ready,
+		Status: metav1.ConditionFalse, Reason: constants.Creating})
+
+	return i.StatusUpdate(ctx, instance)
+}

internal/controller/rekor/actions/to_initialize_phase.go → internal/controller/rekor/actions/transitions/to_initialize_phase.go

@@ -1,4 +1,4 @@
-package actions
+package transitions
 
 import (
 	"context"
@@ -24,6 +24,9 @@ func (i toInitializeAction) Name() string {
 
 func (i toInitializeAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool {
 	c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready)
+	if c == nil {
+		return false
+	}
 	return c.Reason == constants.Creating
 }
 

internal/controller/rekor/rekor_controller.go

@@ -21,15 +21,20 @@ import (
 
 	olpredicate "github.com/operator-framework/operator-lib/predicate"
 	"github.com/securesign/operator/internal/controller/annotations"
-
+	"github.com/securesign/operator/internal/controller/constants"
 	actions2 "github.com/securesign/operator/internal/controller/rekor/actions"
 	backfillredis "github.com/securesign/operator/internal/controller/rekor/actions/backfillRedis"
 	"github.com/securesign/operator/internal/controller/rekor/actions/redis"
 	"github.com/securesign/operator/internal/controller/rekor/actions/server"
+	"github.com/securesign/operator/internal/controller/rekor/actions/transitions"
 	"github.com/securesign/operator/internal/controller/rekor/actions/ui"
 	v13 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	"github.com/securesign/operator/internal/controller/common/action"
 	v12 "k8s.io/api/apps/v1"
@@ -88,6 +93,9 @@ func (r *RekorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 	}
 	target := instance.DeepCopy()
 	actions := []action.Action[rhtasv1alpha1.Rekor]{
+		// register error handler
+		actions2.NewHandleErrorAction(),
+
 		// NONE -> PENDING
 		actions2.NewInitializeConditions(),
 
@@ -96,6 +104,7 @@ func (r *RekorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 		// PENDING -> CREATE
 		server.NewGenerateSignerAction(),
 
+		transitions.NewToCreateAction(),
 		// CREATE
 		actions2.NewRBACAction(),
 		server.NewServerConfigAction(),
@@ -117,7 +126,7 @@ func (r *RekorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 		backfillredis.NewBackfillRedisCronJobAction(),
 
 		// CREATE -> INITIALIZE
-		actions2.NewToInitializeAction(),
+		transitions.NewToInitializeAction(),
 		// INITIALIZE
 		server.NewInitializeAction(),
 		server.NewResolvePubKeyAction(),
@@ -155,7 +164,17 @@ func (r *RekorReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	return ctrl.NewControllerManagedBy(mgr).
 		WithEventFilter(pause).
-		For(&rhtasv1alpha1.Rekor{}).
+		For(&rhtasv1alpha1.Rekor{}, builder.WithPredicates(predicate.Funcs{UpdateFunc: func(event event.UpdateEvent) bool {
+			// do not requeue failed object updates
+			rekor, ok := event.ObjectNew.(*rhtasv1alpha1.Rekor)
+			if !ok {
+				return false
+			}
+			if c := meta.FindStatusCondition(rekor.Status.Conditions, constants.Ready); c != nil {
+				return c.Reason != constants.Failure
+			}
+			return true
+		}})).
 		Owns(&v12.Deployment{}).
 		Owns(&v13.Service{}).
 		Owns(&v1.Ingress{}).