Improve logs based on pod conditions

ray-project · Aug 30, 2024 · 9f67eae · 9f67eae
1 parent 3fbce0f
commit 9f67eae
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/ray-operator/controllers/ray/raycluster_controller.go b/ray-operator/controllers/ray/raycluster_controller.go
@@ -692,6 +692,21 @@ func (r *RayClusterReconciler) reconcilePods(ctx context.Context, instance *rayv
 		deleted := struct{}{}
 		numDeletedUnhealthyWorkerPods := 0
 		for _, workerPod := range workerPods.Items {
+			logger.Info("reconcilePods", "Worker Pod Name: ", workerPod.Name, " Status: ", workerPod.Status.Phase)
+			if workerPod.Status.Phase != "Running" {
+				if len(workerPod.Status.Conditions) > 0 {
+					for _, condition := range workerPod.Status.Conditions {
+						if condition.Message != "" {
+							logger.Info("reconcilePods",
+								"Worker Pod Name", workerPod.Name,
+								"Status", workerPod.Status.Phase,
+								"Condition Type", condition.Type,
+								"Condition Status", condition.Status,
+								"Message", condition.Message)
+						}
+					}
+				}
+			}
 			shouldDelete, reason := shouldDeletePod(workerPod, rayv1.WorkerNode)
 			logger.Info("reconcilePods", "worker Pod", workerPod.Name, "shouldDelete", shouldDelete, "reason", reason)
 			if shouldDelete {