Skip to content

Commit

Permalink
account for new generation and deleted
Browse files Browse the repository at this point in the history
  • Loading branch information
@isaaguilar
isaaguilar committed Oct 25, 2023
1 parent 5de275c commit 8c5c2d3
Showing 1 changed file with 47 additions and 41 deletions.
88 changes: 47 additions & 41 deletions pkg/controllers/terraform_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,34 +588,6 @@ func (r *ReconcileTerraform) Reconcile(ctx context.Context, request reconcile.Re
return reconcile.Result{}, err
}

if tf.Labels != nil {
if label, found := tf.Labels["kubernetes.io/change-cause"]; found {
retry := false
if tf.Status.RetryEventReason == nil {
retry = true
} else if *tf.Status.RetryEventReason != label {
retry = true
}

if retry {
now := metav1.Now()
tf.Status.RetryEventReason = &label
tf.Status.RetryTimestamp = &now
taskType := tfv1beta1.RunInit
if strings.HasSuffix(label, ".setup") {
taskType = tfv1beta1.RunSetup
}
stage := newStage(tf, taskType, label, isTaskInterruptable(taskType), tfv1beta1.StateInitializing)
tf.Status.Stage = *stage
err := r.updateStatusWithRetry(ctx, tf, &tf.Status, reqLogger)
if err != nil {
reqLogger.V(1).Info(err.Error())
}
return reconcile.Result{}, nil
}
}
}

// Final delete by removing finalizers
if tf.Status.Phase == tfv1beta1.PhaseDeleted {
reqLogger.Info("Remove finalizers")
Expand Down Expand Up @@ -698,14 +670,33 @@ func (r *ReconcileTerraform) Reconcile(ctx context.Context, request reconcile.Re
//
// }
// }
stage := r.checkSetNewStage(ctx, tf)

retry := false
if tf.Labels != nil {
if label, found := tf.Labels["kubernetes.io/change-cause"]; found {

if tf.Status.RetryEventReason == nil {
retry = true
} else if *tf.Status.RetryEventReason != label {
retry = true
}

if retry {
now := metav1.Now()
tf.Status.RetryEventReason = &label // saved via updateStatusWithRetry
tf.Status.RetryTimestamp = &now // saved via updateStatusWithRetry
}
}
}

stage := r.checkSetNewStage(ctx, tf, retry)
if stage != nil {
tf.Status.Stage = *stage
if stage.Reason == "RESTARTED_WORKFLOW" || stage.Reason == "RESTARTED_DELETE_WORKFLOW" {
_ = r.removeOldPlan(tf)
_ = r.removeOldPlan(tf.Namespace, tf.Name, tf.Status.Stage.Reason, tf.Generation)
// TODO what to do if the remove old plan function fails
}
reqLogger.V(2).Info(fmt.Sprintf("Stage moving from '%s' -> '%s'", tf.Status.Stage.TaskType, stage.TaskType))
tf.Status.Stage = *stage
desiredStatus := tf.Status
err := r.updateStatusWithRetry(ctx, tf, &desiredStatus, reqLogger)
if err != nil {
Expand Down Expand Up @@ -889,7 +880,7 @@ func (r *ReconcileTerraform) Reconcile(ctx context.Context, request reconcile.Re
reqLogger.V(1).Info(fmt.Sprintf("Setting up the '%s' pod", podType))
err := r.setupAndRun(ctx, tf, runOpts)
if err != nil {
reqLogger.Error(err, "")
reqLogger.Error(err, err.Error())
return reconcile.Result{}, err
}
if tf.Status.Phase == tfv1beta1.PhaseInitializing {
Expand Down Expand Up @@ -1068,7 +1059,7 @@ func getConfiguredTasks(taskOptions *[]tfv1beta1.TaskOption) []tfv1beta1.TaskNam
// When a stage has already triggered a pod, the only way for the pod to transition to the next stage is for
// the pod to complete successfully. Any other pod phase will keep the pod in the current stage, or in the
// case of the apply task, the workflow will be restarted.
func (r ReconcileTerraform) checkSetNewStage(ctx context.Context, tf *tfv1beta1.Terraform) *tfv1beta1.Stage {
func (r ReconcileTerraform) checkSetNewStage(ctx context.Context, tf *tfv1beta1.Terraform, isRetry bool) *tfv1beta1.Stage {
var isNewStage bool
var podType tfv1beta1.TaskName
var reason string
Expand All @@ -1090,8 +1081,23 @@ func (r ReconcileTerraform) checkSetNewStage(ctx context.Context, tf *tfv1beta1.
currentStageIsRunning := currentStage.State == tfv1beta1.StateInProgress
isNewGeneration := currentStage.Generation != tf.Generation

// resource status
if currentStageCanNotBeInterrupted && currentStageIsRunning {
if isRetry && !isToBeDeletedOrIsDeleting && !isNewGeneration {
isNewStage = true
reason = *tf.Status.RetryEventReason
podType = tfv1beta1.RunInit
if strings.HasSuffix(reason, ".setup") {
podType = tfv1beta1.RunSetup
}
interruptible = isTaskInterruptable(podType)
} else if isRetry && isToBeDeletedOrIsDeleting && !isNewGeneration {
isNewStage = true
reason = *tf.Status.RetryEventReason
podType = tfv1beta1.RunInitDelete
if strings.HasSuffix(reason, ".setup") {
podType = tfv1beta1.RunSetupDelete
}
interruptible = isTaskInterruptable(podType)
} else if currentStageCanNotBeInterrupted && currentStageIsRunning {
// Cannot change to the next stage because the current stage cannot be
// interrupted and is currently running
isNewStage = false
Expand Down Expand Up @@ -1163,20 +1169,20 @@ func (r ReconcileTerraform) checkSetNewStage(ctx context.Context, tf *tfv1beta1.

}

func (r ReconcileTerraform) removeOldPlan(tf *tfv1beta1.Terraform) error {
func (r ReconcileTerraform) removeOldPlan(namespace, name, reason string, generation int64) error {
labelSelectors := []string{
fmt.Sprintf("terraforms.tf.galleybytes.com/generation==%d", tf.Generation),
fmt.Sprintf("terraforms.tf.galleybytes.com/resourceName=%s", tf.Name),
fmt.Sprintf("terraforms.tf.galleybytes.com/generation==%d", generation),
fmt.Sprintf("terraforms.tf.galleybytes.com/resourceName=%s", name),
"app.kubernetes.io/instance",
}
if tf.Status.Stage.Reason == "RESTARTED_WORKFLOW" {
if reason == "RESTARTED_WORKFLOW" {
labelSelectors = append(labelSelectors, []string{
fmt.Sprintf("app.kubernetes.io/instance!=%s", tfv1beta1.RunSetup),
fmt.Sprintf("app.kubernetes.io/instance!=%s", tfv1beta1.RunPreInit),
fmt.Sprintf("app.kubernetes.io/instance!=%s", tfv1beta1.RunInit),
fmt.Sprintf("app.kubernetes.io/instance!=%s", tfv1beta1.RunPostInit),
}...)
} else if tf.Status.Stage.Reason == "RESTARTED_DELETE_WORKFLOW" {
} else if reason == "RESTARTED_DELETE_WORKFLOW" {
labelSelectors = append(labelSelectors, []string{
fmt.Sprintf("app.kubernetes.io/instance!=%s", tfv1beta1.RunSetupDelete),
fmt.Sprintf("app.kubernetes.io/instance!=%s", tfv1beta1.RunPreInitDelete),
Expand All @@ -1195,7 +1201,7 @@ func (r ReconcileTerraform) removeOldPlan(tf *tfv1beta1.Terraform) error {
err = r.Client.DeleteAllOf(context.TODO(), &corev1.Pod{}, &client.DeleteAllOfOptions{
ListOptions: client.ListOptions{
LabelSelector: labelSelector,
Namespace: tf.Namespace,
Namespace: namespace,
FieldSelector: fieldSelector,
},
})
Expand Down

0 comments on commit 8c5c2d3

Please sign in to comment.