-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SECURESIGN-1047] CTLog error handler
- Loading branch information
Showing
8 changed files
with
283 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package actions | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" | ||
| "github.com/securesign/operator/internal/controller/common/action" | ||
| "github.com/securesign/operator/internal/controller/constants" | ||
| v1 "k8s.io/api/core/v1" | ||
| "k8s.io/apimachinery/pkg/api/meta" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| func NewHandleErrorAction() action.Action[rhtasv1alpha1.CTlog] { | ||
| return &handleErrorAction{} | ||
| } | ||
|
|
||
| type handleErrorAction struct { | ||
| action.BaseAction | ||
| } | ||
|
|
||
| func (i handleErrorAction) Name() string { | ||
| return "error handler" | ||
| } | ||
|
|
||
| func (i handleErrorAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.CTlog) bool { | ||
| c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) | ||
| if c == nil { | ||
| return false | ||
| } | ||
| return c.Reason == constants.Failure && instance.Status.Restarts < constants.AllowedRestarts | ||
| } | ||
|
|
||
| func (i handleErrorAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { | ||
| i.Recorder.Event(instance, v1.EventTypeWarning, constants.Failure, "Restarted by error handler") | ||
|
|
||
| newStatus := rhtasv1alpha1.CTlogStatus{} | ||
|
|
||
| newStatus.Restarts = instance.Status.Restarts + 1 | ||
| if newStatus.Restarts == constants.AllowedRestarts { | ||
| meta.SetStatusCondition(&newStatus.Conditions, metav1.Condition{ | ||
| Type: constants.Ready, | ||
| Status: metav1.ConditionFalse, | ||
| Reason: constants.Failure, | ||
| Message: "Restart threshold reached", | ||
| }) | ||
| instance.Status = newStatus | ||
| return i.StatusUpdate(ctx, instance) | ||
| } | ||
|
|
||
| // - keep the status.treeId if not nil | ||
| newStatus.TreeID = instance.Status.TreeID | ||
|
|
||
| newStatus.PrivateKeyRef = instance.Status.PrivateKeyRef.DeepCopy() | ||
|
|
||
| newStatus.PublicKeyRef = instance.Status.PublicKeyRef.DeepCopy() | ||
|
|
||
| newStatus.PrivateKeyPasswordRef = instance.Status.PrivateKeyPasswordRef.DeepCopy() | ||
|
|
||
| if meta.IsStatusConditionTrue(instance.Status.Conditions, CertCondition) { | ||
| copy(newStatus.RootCertificates, instance.Status.RootCertificates) | ||
| } | ||
|
|
||
| if meta.IsStatusConditionTrue(instance.Status.Conditions, ServerCondition) { | ||
| instance.Status.ServerConfigRef.DeepCopyInto(newStatus.ServerConfigRef) | ||
| // do not append server condition - let controller to redeploy | ||
| } | ||
|
|
||
| meta.SetStatusCondition(&newStatus.Conditions, metav1.Condition{ | ||
| Type: constants.Ready, | ||
| Status: metav1.ConditionFalse, | ||
| Reason: constants.Pending, | ||
| Message: "Restarted by error handler", | ||
| }) | ||
| instance.Status = newStatus | ||
| return i.StatusUpdate(ctx, instance) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| /* | ||
| Copyright 2023. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package ctlog | ||
|
|
||
| import ( | ||
| "context" | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/securesign/operator/internal/controller/common/utils" | ||
| "github.com/securesign/operator/internal/controller/common/utils/kubernetes" | ||
| fulcio "github.com/securesign/operator/internal/controller/fulcio/actions" | ||
| trillian "github.com/securesign/operator/internal/controller/trillian/actions" | ||
| appsv1 "k8s.io/api/apps/v1" | ||
| runtimeClient "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| "github.com/securesign/operator/api/v1alpha1" | ||
| "github.com/securesign/operator/internal/controller/constants" | ||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/apimachinery/pkg/api/errors" | ||
| "k8s.io/apimachinery/pkg/api/meta" | ||
|
|
||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/types" | ||
| ) | ||
|
|
||
| var _ = Describe("CTlog ErrorHandler", func() { | ||
| Context("CTlog ErrorHandler test", func() { | ||
|
|
||
| const ( | ||
| Name = "test" | ||
| Namespace = "errorhandler" | ||
| ) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| namespace := &corev1.Namespace{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: Namespace, | ||
| }, | ||
| } | ||
|
|
||
| typeNamespaceName := types.NamespacedName{Name: Name, Namespace: Namespace} | ||
| instance := &v1alpha1.CTlog{} | ||
|
|
||
| BeforeEach(func() { | ||
| // workaround - disable "host" mode in CreateTrillianTree function | ||
| Expect(os.Setenv("CONTAINER_MODE", "true")).To(Not(HaveOccurred())) | ||
|
|
||
| By("Creating the Namespace to perform the tests") | ||
| err := k8sClient.Create(ctx, namespace) | ||
| Expect(err).To(Not(HaveOccurred())) | ||
| }) | ||
|
|
||
| AfterEach(func() { | ||
| By("removing the custom resource for the Kind CTlog") | ||
| found := &v1alpha1.CTlog{} | ||
| err := k8sClient.Get(ctx, typeNamespaceName, found) | ||
| Expect(err).To(Not(HaveOccurred())) | ||
|
|
||
| Eventually(func() error { | ||
| return k8sClient.Delete(context.TODO(), found) | ||
| }, 2*time.Minute, time.Second).Should(Succeed()) | ||
|
|
||
| // TODO(user): Attention if you improve this code by adding other context test you MUST | ||
| // be aware of the current delete namespace limitations. | ||
| // More info: https://book.kubebuilder.io/reference/envtest.html#testing-considerations | ||
| By("Deleting the Namespace to perform the tests") | ||
| _ = k8sClient.Delete(ctx, namespace) | ||
| }) | ||
|
|
||
| It("should successfully reconcile a custom resource for CTlog", func() { | ||
| By("creating the custom resource for the Kind CTlog") | ||
| err := k8sClient.Get(ctx, typeNamespaceName, instance) | ||
| if err != nil && errors.IsNotFound(err) { | ||
| // Let's mock our custom resource at the same way that we would | ||
| // apply on the cluster the manifest under config/samples | ||
| instance := &v1alpha1.CTlog{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: Name, | ||
| Namespace: Namespace, | ||
| }, | ||
| Spec: v1alpha1.CTlogSpec{}, | ||
| } | ||
| err = k8sClient.Create(ctx, instance) | ||
| Expect(err).To(Not(HaveOccurred())) | ||
| } | ||
|
|
||
| Expect(k8sClient.Create(ctx, kubernetes.CreateSecret("test", Namespace, | ||
| map[string][]byte{"cert": []byte("fakeCert")}, | ||
| map[string]string{fulcio.FulcioCALabel: "cert"}, | ||
| ))).To(Succeed()) | ||
|
|
||
| Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed()) | ||
|
|
||
| found := &v1alpha1.CTlog{} | ||
|
|
||
| By("Deployment should fail") | ||
| Eventually(func() string { | ||
|
|
||
| Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) | ||
| condition := meta.FindStatusCondition(found.Status.Conditions, constants.Ready) | ||
| if condition == nil { | ||
| return "" | ||
| } | ||
| return condition.Reason | ||
| }).Should(Equal(constants.Failure)) | ||
|
|
||
| key := found.Status.PrivateKeyRef.Name | ||
| Expect(key).To(Not(BeEmpty())) | ||
|
|
||
| By("Periodically trying to restart deployment") | ||
| Eventually(func() string { | ||
| found := &v1alpha1.CTlog{} | ||
| Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) | ||
| return meta.FindStatusCondition(found.Status.Conditions, constants.Ready).Reason | ||
| }).Should(Not(Equal(constants.Failure))) | ||
| Eventually(func() string { | ||
| found := &v1alpha1.CTlog{} | ||
| Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) | ||
| return meta.FindStatusCondition(found.Status.Conditions, constants.Ready).Reason | ||
| }).Should(Equal(constants.Failure)) | ||
|
|
||
| By("After fixing the problem the CTlog instance is Ready") | ||
| Eventually(func() error { | ||
| Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) | ||
| found.Spec.TreeID = utils.Pointer(int64(1)) | ||
| return k8sClient.Update(ctx, found) | ||
| }).Should(Succeed()) | ||
|
|
||
| By("Waiting until CTlog instance is Initialization") | ||
| Eventually(func() string { | ||
| found := &v1alpha1.CTlog{} | ||
| Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) | ||
| return meta.FindStatusCondition(found.Status.Conditions, constants.Ready).Reason | ||
| }).Should(Equal(constants.Initialize)) | ||
|
|
||
| deployments := &appsv1.DeploymentList{} | ||
| Expect(k8sClient.List(ctx, deployments, runtimeClient.InNamespace(Namespace))).To(Succeed()) | ||
| By("Move to Ready phase") | ||
| for _, d := range deployments.Items { | ||
| d.Status.Conditions = []appsv1.DeploymentCondition{ | ||
| {Status: corev1.ConditionTrue, Type: appsv1.DeploymentAvailable, Reason: constants.Ready}} | ||
| Expect(k8sClient.Status().Update(ctx, &d)).Should(Succeed()) | ||
| } | ||
| // Workaround to succeed condition for Ready phase | ||
|
|
||
| Eventually(func() bool { | ||
| found := &v1alpha1.CTlog{} | ||
| Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) | ||
| return meta.IsStatusConditionTrue(found.Status.Conditions, constants.Ready) | ||
| }).Should(BeTrue()) | ||
|
|
||
| By("Pregenerated resources are reused") | ||
| Expect(key).To(Equal(found.Status.PrivateKeyRef.Name)) | ||
| }) | ||
| }) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters