feat(recipes): deepen 21 chainsaw health checks; close epic #660

recipes/checks/agentgateway/health-check.yaml

@@ -41,9 +41,13 @@ spec:
                 (availableReplicas > `0`): true
     - name: validate-all-pods-healthy
       try:
-        # Assert no pods are in unhealthy phases.
-        # Pods must be Running (long-lived) or Succeeded (completed jobs).
-        # This catches Pending (init containers, scheduling), Failed, and Unknown.
+        # Assert no pods are in unhealthy phases (Pending / Failed /
+        # Unknown) or stuck in Running phase with a known-bad
+        # container-waiting reason (CrashLoopBackOff,
+        # ImagePullBackOff, ErrImagePull, CreateContainerConfigError).
+        # Phase-only filtering misses CrashLoopBackOff because such
+        # pods typically remain in Running phase with an unhealthy
+        # container.
         - error:
             resource:
               apiVersion: v1
@@ -68,3 +72,27 @@ spec:
                 namespace: agentgateway-system
               status:
                 phase: Unknown
+        # JMESPath projection (not single-element match): kyverno-json's
+        # sliceNode.Assert gates on exact slice length before per-element
+        # comparison, so a `containerStatuses: - state: ...` shape only
+        # matches single-container pods and silently passes sidecars. Use
+        # `containerStatuses[?...] | length(@) > 0` to fire on any
+        # container at any index. Parallel initContainerStatuses block
+        # below catches init-container failures (invisible to
+        # containerStatuses). Per PR #1245 review (@ArangoGutierrez).
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: agentgateway-system
+              status:
+                (containerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: agentgateway-system
+              status:
+                (initContainerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true

recipes/checks/aws-ebs-csi-driver/health-check.yaml

@@ -42,7 +42,13 @@ spec:
                 (availableReplicas > `0`): true
     - name: validate-all-pods-healthy
       try:
-        # Assert no EBS CSI driver pods are in unhealthy phases.
+        # Assert no EBS CSI driver pods are in unhealthy phases
+        # (Pending / Failed / Unknown) or stuck in Running phase
+        # with a known-bad container-waiting reason
+        # (CrashLoopBackOff, ImagePullBackOff, ErrImagePull,
+        # CreateContainerConfigError). Phase-only filtering misses
+        # CrashLoopBackOff because such pods typically remain in
+        # Running phase with an unhealthy container.
         # Uses label selector because kube-system is a shared namespace.
         - error:
             resource:
@@ -74,3 +80,31 @@ spec:
                   app.kubernetes.io/name: aws-ebs-csi-driver
               status:
                 phase: Unknown
+        # JMESPath projection (not single-element match): kyverno-json's
+        # sliceNode.Assert gates on exact slice length before per-element
+        # comparison, so a `containerStatuses: - state: ...` shape only
+        # matches single-container pods and silently passes sidecars. Use
+        # `containerStatuses[?...] | length(@) > 0` to fire on any
+        # container at any index. Parallel initContainerStatuses block
+        # below catches init-container failures (invisible to
+        # containerStatuses). Per PR #1245 review (@ArangoGutierrez).
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: kube-system
+                labels:
+                  app.kubernetes.io/name: aws-ebs-csi-driver
+              status:
+                (containerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: kube-system
+                labels:
+                  app.kubernetes.io/name: aws-ebs-csi-driver
+              status:
+                (initContainerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true

recipes/checks/aws-efa/health-check.yaml

@@ -14,11 +14,15 @@
 
 # AWS EFA Health Check
 #
-# Validates that the AWS EFA K8s device plugin is running and healthy in the
-# kube-system namespace. Checks that the aws-efa-k8s-device-plugin DaemonSet
-# has at least one ready pod and that no EFA plugin pods (selected by label)
-# are stuck in Pending, Failed, or Unknown phases. Label selectors are used
-# because kube-system is a shared namespace.
+# Validates that the AWS EFA K8s device plugin is running and healthy
+# in the kube-system namespace. The DaemonSet must roll out fully —
+# numberReady == desiredNumberScheduled (with desiredNumberScheduled > 0
+# as a guard against vacuous pass on a 0-node selector). The previous
+# check gated only on numberReady > 0, which passed on partial
+# failures; see #1222 / recipes/checks/nfd/health-check.yaml for the
+# same rationale. Container-state errors (CrashLoopBackOff,
+# ImagePullBackOff, ErrImagePull, CreateContainerConfigError) are
+# label-scoped because kube-system is a shared namespace.
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
@@ -27,10 +31,8 @@ spec:
   timeouts:
     assert: 5m
   steps:
-    - name: validate-deployment-exists
+    - name: validate-daemonset-fully-rolled-out
       try:
-        # Guard against vacuous pass: verify the aws-efa-k8s-device-plugin
-        # DaemonSet exists and has at least one ready pod.
         - assert:
             resource:
               apiVersion: apps/v1
@@ -39,11 +41,10 @@ spec:
                 name: aws-efa-k8s-device-plugin
                 namespace: kube-system
               status:
-                (numberReady > `0`): true
+                (desiredNumberScheduled > `0`): true
+                (numberReady == desiredNumberScheduled): true
     - name: validate-all-pods-healthy
       try:
-        # Assert no EFA plugin pods are in unhealthy phases.
-        # Uses label selector because kube-system is a shared namespace.
         - error:
             resource:
               apiVersion: v1
@@ -74,3 +75,31 @@ spec:
                   app.kubernetes.io/name: aws-efa-k8s-device-plugin
               status:
                 phase: Unknown
+        # JMESPath projection (not single-element match): kyverno-json's
+        # sliceNode.Assert gates on exact slice length before per-element
+        # comparison, so a `containerStatuses: - state: ...` shape only
+        # matches single-container pods and silently passes sidecars. Use
+        # `containerStatuses[?...] | length(@) > 0` to fire on any
+        # container at any index. Parallel initContainerStatuses block
+        # below catches init-container failures (invisible to
+        # containerStatuses). Per PR #1245 review (@ArangoGutierrez).
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: kube-system
+                labels:
+                  app.kubernetes.io/name: aws-efa-k8s-device-plugin
+              status:
+                (containerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: kube-system
+                labels:
+                  app.kubernetes.io/name: aws-efa-k8s-device-plugin
+              status:
+                (initContainerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true

recipes/checks/cert-manager/health-check.yaml

@@ -15,9 +15,16 @@
 # Cert Manager Health Check
 #
 # Validates that cert-manager is running and healthy in the cert-manager
-# namespace. Checks that the cert-manager deployment has at least one
-# available replica and that no pods in the namespace are stuck in Pending,
-# Failed, or Unknown phases.
+# namespace. The chart deploys three Deployments — the controller
+# (cert-manager), the CA injector (cert-manager-cainjector), and the
+# webhook (cert-manager-webhook) — all of which must be available for
+# the operator to issue / inject / validate certificates. The previous
+# check only asserted the controller; this check covers all three to
+# avoid silently passing while the webhook is down (which would block
+# every Certificate/Issuer CR admission).
+#
+# Also asserts the three core CRDs are Established=True so the
+# apiserver will accept CRs of those kinds (per #1222 / #660).
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
@@ -26,10 +33,37 @@ spec:
   timeouts:
     assert: 5m
   steps:
-    - name: validate-deployment-exists
+    - name: validate-crds-established
+      try:
+        - assert:
+            resource:
+              apiVersion: apiextensions.k8s.io/v1
+              kind: CustomResourceDefinition
+              metadata:
+                name: certificates.cert-manager.io
+              status:
+                (conditions[?type == 'Established']):
+                  - status: "True"
+        - assert:
+            resource:
+              apiVersion: apiextensions.k8s.io/v1
+              kind: CustomResourceDefinition
+              metadata:
+                name: issuers.cert-manager.io
+              status:
+                (conditions[?type == 'Established']):
+                  - status: "True"
+        - assert:
+            resource:
+              apiVersion: apiextensions.k8s.io/v1
+              kind: CustomResourceDefinition
+              metadata:
+                name: clusterissuers.cert-manager.io
+              status:
+                (conditions[?type == 'Established']):
+                  - status: "True"
+    - name: validate-controller-deployment
       try:
-        # Guard against vacuous pass on empty namespace: verify the
-        # cert-manager deployment exists and has at least one ready replica.
         - assert:
             resource:
               apiVersion: apps/v1
@@ -39,11 +73,34 @@ spec:
                 namespace: cert-manager
               status:
                 (availableReplicas > `0`): true
+    - name: validate-cainjector-deployment
+      try:
+        - assert:
+            resource:
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: cert-manager-cainjector
+                namespace: cert-manager
+              status:
+                (availableReplicas > `0`): true
+    - name: validate-webhook-deployment
+      try:
+        # The webhook must be ready before any Certificate / Issuer CR
+        # can be admitted. Without this assert, a healthy controller
+        # with a dead webhook would still pass the check while the
+        # cluster silently rejects every cert-manager CR.
+        - assert:
+            resource:
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: cert-manager-webhook
+                namespace: cert-manager
+              status:
+                (availableReplicas > `0`): true
     - name: validate-all-pods-healthy
       try:
-        # Assert no pods are in unhealthy phases.
-        # Pods must be Running (long-lived) or Succeeded (completed jobs).
-        # This catches Pending (init containers, scheduling), Failed, and Unknown.
         - error:
             resource:
               apiVersion: v1
@@ -68,3 +125,27 @@ spec:
                 namespace: cert-manager
               status:
                 phase: Unknown
+        # JMESPath projection (not single-element match): kyverno-json's
+        # sliceNode.Assert gates on exact slice length before per-element
+        # comparison, so a `containerStatuses: - state: ...` shape only
+        # matches single-container pods and silently passes sidecars. Use
+        # `containerStatuses[?...] | length(@) > 0` to fire on any
+        # container at any index. Parallel initContainerStatuses block
+        # below catches init-container failures (invisible to
+        # containerStatuses). Per PR #1245 review (@ArangoGutierrez).
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: cert-manager
+              status:
+                (containerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: cert-manager
+              status:
+                (initContainerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true

recipes/checks/dynamo-platform/health-check.yaml

@@ -15,9 +15,12 @@
 # Dynamo Platform Health Check
 #
 # Validates that the Dynamo Platform is running and healthy in the
-# dynamo-system namespace. Checks that the dynamo-operator deployment has
-# at least one available replica and that no pods in the namespace are stuck
-# in Pending, Failed, or Unknown phases.
+# dynamo-system namespace:
+# 1. dynamocomponentdeployments.nvidia.com CRD Established=True (the
+#    apiserver will accept DCD CRs the operator manages).
+# 2. dynamo-operator Deployment has at least one available replica.
+# 3. No pods stuck in Pending/Failed/Unknown phases or known-bad
+#    container-waiting states.
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
@@ -26,10 +29,19 @@ spec:
   timeouts:
     assert: 5m
   steps:
+    - name: validate-dcd-crd-established
+      try:
+        - assert:
+            resource:
+              apiVersion: apiextensions.k8s.io/v1
+              kind: CustomResourceDefinition
+              metadata:
+                name: dynamocomponentdeployments.nvidia.com
+              status:
+                (conditions[?type == 'Established']):
+                  - status: "True"
     - name: validate-deployment-exists
       try:
-        # Guard against vacuous pass on empty namespace: verify the
-        # dynamo-operator deployment exists and has at least one ready replica.
         - assert:
             resource:
               apiVersion: apps/v1
@@ -41,9 +53,6 @@ spec:
                 (availableReplicas > `0`): true
     - name: validate-all-pods-healthy
       try:
-        # Assert no pods are in unhealthy phases.
-        # Pods must be Running (long-lived) or Succeeded (completed jobs).
-        # This catches Pending (init containers, scheduling), Failed, and Unknown.
         - error:
             resource:
               apiVersion: v1
@@ -68,3 +77,27 @@ spec:
                 namespace: dynamo-system
               status:
                 phase: Unknown
+        # JMESPath projection (not single-element match): kyverno-json's
+        # sliceNode.Assert gates on exact slice length before per-element
+        # comparison, so a `containerStatuses: - state: ...` shape only
+        # matches single-container pods and silently passes sidecars. Use
+        # `containerStatuses[?...] | length(@) > 0` to fire on any
+        # container at any index. Parallel initContainerStatuses block
+        # below catches init-container failures (invisible to
+        # containerStatuses). Per PR #1245 review (@ArangoGutierrez).
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: dynamo-system
+              status:
+                (containerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true
+        - error:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                namespace: dynamo-system
+              status:
+                (initContainerStatuses[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true