[service-proxy-probe] Monitoring service-proxy endpoints using BlackboxExporter

kube-monitoring/charts/templates/probe-plugin-exposed-services.yaml

@@ -0,0 +1,103 @@
+{{- /*
+  This template creates Probe CRDs for monitoring plugin exposed services.
+
+  It uses Helm's lookup function to discover plugins with exposed services.
+  The lookup will return empty if:
+  - The Plugin CRD doesn't exist on this cluster
+  - No plugins are found in the namespace
+
+  This is expected behavior - the template gracefully handles missing CRDs.
+
+  Namespace Restriction:
+  - This template ONLY runs when Release.Namespace matches organizationName
+  - For example: if organizationName=sci, Probes are only created in sci namespace
+  - When deployed to other namespaces (e.g., greenhouse), this template is silently skipped
+
+  Configuration:
+  - Requires .Values.blackboxExporter.enabled to be true
+  - Set .Values.pluginProbe.enabled to false to disable entirely
+  - Set .Values.pluginProbe.pluginDefinition to filter by plugin type
+*/ -}}
+
+# Get organization namespace - this is where Plugins are located 
+{{- $orgNamespace := .Values.global.greenhouse.organizationName | default "demo" }}
+
+# FIRST CHECK: Only proceed if Release.Namespace matches organizationName
+# This ensures Probes are only created in the organization namespace
+{{- if eq $.Release.Namespace $orgNamespace }}
+
+# SECOND CHECK: Blackbox exporter must be enabled
+{{- if .Values.blackboxExporter.enabled }}
+
+# Check if plugin probe is enabled (defaults to true if not specified)
+{{- $pluginProbeEnabled := true }}
+{{- if hasKey .Values "pluginProbe" }}
+{{- if hasKey .Values.pluginProbe "enabled" }}
+{{- $pluginProbeEnabled = .Values.pluginProbe.enabled }}
+{{- end }}
+{{- end }}
+
+{{- if $pluginProbeEnabled }}
+
+# Get pluginDefinition filter from values (empty string means no filter)
+{{- $pluginDefinitionFilter := .Values.pluginProbe.pluginDefinition | default "kube-monitoring" }}
+
+# Safely attempt lookup - returns empty map if CRD doesn't exist
+{{- $plugins := lookup "greenhouse.sap/v1alpha1" "Plugin" $orgNamespace "" | default dict }}
+{{- if and $plugins $plugins.items }}
+{{- range $plugins.items }}
+
+# Filter: Only process Plugins matching pluginDefinition (if filter is set)
+{{- if (eq .spec.pluginDefinitionRef.name $pluginDefinitionFilter) }}
+{{- if .status }}
+{{- if .status.exposedServices }}
+{{- $plugin := . }}
+{{- $clusterName := .spec.clusterName | default "unknown" }}
+
+# Create probes for all plugins with exposed services
+{{- range $url, $service := .status.exposedServices }}
+---
+# Probe for plugin {{ $plugin.metadata.name }} on cluster {{ $clusterName }} - service {{ $service.name }}
+apiVersion: monitoring.coreos.com/v1
+kind: Probe
+metadata:
+  name: plugin-{{ $plugin.metadata.name }}-{{ $service.name }}
+  namespace: {{ $.Release.Namespace }}
+  annotations:
+    helm.sh/resource-policy: keep
+  labels:
+    app: plugin-probe
+    plugin: kube-monitoring
+    component: plugin-monitoring
+    release: {{ $.Release.Name }}
+    cluster: {{ $clusterName }}
+    service: {{ $service.name }}
+spec:
+  interval: 30s
+  scrapeTimeout: 10s
+  prober:
+    url: {{ $.Release.Name }}-blackbox-exporter.{{ $.Release.Namespace }}.svc.cluster.local:9115
+    path: /probe
+  module: http_2xx
+  targets:
+    staticConfig:
+      static:
+      - {{ $url }}
+      labels:
+        job: plugin-exposed-services
+        cluster: {{ $clusterName }}
+        plugin_name: {{ $plugin.metadata.name }}
+  metricRelabelings:
+  - sourceLabels: [instance]
+    targetLabel: target
+  - regex: instance
+    action: labeldrop
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

kube-monitoring/charts/values.yaml

@@ -3,6 +3,27 @@ global:
   # This can be used to add a `support_group` or `service` label to all resources and alerting rules.
   # @section -- Global options
   commonLabels: {}
+
+  # Greenhouse-specific configuration
+  greenhouse:
+    # -- Organization name for Greenhouse Plugin deployment
+    # Used by probe-plugin-exposed-services.yaml template to discover Plugins in the organization namespace
+    # IMPORTANT: The probe template only creates Probes when installing in the organization namespace.
+    # When installing in other namespaces (e.g., greenhouse), probes will be automatically skipped.
+    # @section -- Global options
+    organizationName: "sci"
+
+    # IMPORTANT NOTE ON MULTI-NAMESPACE INSTALLATIONS:
+    # When installing this chart in multiple namespaces, you MUST use different release names
+    # to avoid ClusterRole conflicts from the kube-state-metrics subchart.
+    # 
+    # Example:
+    #   helm install kube-monitoring-sci ./charts --namespace sci
+    #   helm install kube-monitoring-greenhouse ./charts --namespace greenhouse
+    # 
+    # The probe-plugin-exposed-services.yaml template will automatically:
+    # - Create Probes when Release.Namespace == organizationName (e.g., sci)
+    # - Skip Probe creation when Release.Namespace != organizationName (e.g., greenhouse)
 
 # kube-prometheus-stack configuration scoped to kube-monitoring
 kubeMonitoring:
@@ -520,7 +541,7 @@ testFramework:
 blackboxExporter:
   # -- To enable Blackbox Exporter (supported probers: grpc-prober)
   # @section -- Blackbox exporter config
-  enabled: false
+  enabled: true
 
   # @ignored
   secretConfig: true
@@ -535,8 +556,16 @@ blackboxExporter:
             ca_file: /etc/blackbox/tls/ca.crt
             cert_file: /etc/blackbox/tls/tls.crt
             key_file: /etc/blackbox/tls/tls.key
-
-  # -- TLS secret of the Thanos global instance to mount for probing, mandatory for using Blackbox exporter.
+      http_2xx:
+        prober: http
+        timeout: 5s
+        http:
+          valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+          valid_status_codes: []
+          method: GET
+          preferred_ip_protocol: "ip4"
+
+  # -- TLS secret of the Thanos global instance to mount for probing, mandatory for using Blackbox exporter with gRPC.
   # @default -- <pre>- name: blackbox-exporter-tls<br>  secret:<br>    defaultMode: 420<br>    secretName: \<secretName\></pre>
   # @section -- Blackbox exporter config
   extraVolumes: []
@@ -545,10 +574,37 @@ blackboxExporter:
     #     defaultMode: 420
     #     secretName: <secretName>
   # @ignored
+  # extraVolumeMounts: []
+  # # Example:
   extraVolumeMounts:
     - name: blackbox-exporter-tls
       mountPath: /etc/blackbox/tls
 
+# Plugin exposed services monitoring configuration
+# This section is used by the probe-plugin-exposed-services.yaml template to automatically
+# generate Probe CRDs for monitoring Plugin exposed services discovered via Helm lookup
+pluginProbe:
+  # -- Enable automatic Probe generation for Plugin exposed services
+  # @section -- Plugin monitoring options
+  enabled: true
+
+  # -- Filter Plugins by pluginDefinition field. Only Plugins matching this value will have Probes created.
+  # Set to empty string "" to monitor all Plugins regardless of pluginDefinition.
+  # @section -- Plugin monitoring options
+  pluginDefinition: "kube-monitoring"
+
+  # -- Probe interval for Plugin exposed service checks
+  # @section -- Plugin monitoring options
+  interval: "30s"
+
+  # -- Scrape timeout for Plugin exposed service checks
+  # @section -- Plugin monitoring options
+  scrapeTimeout: "10s"
+
+  # -- Blackbox exporter module to use for probing
+  # @section -- Plugin monitoring options
+  module: "http_2xx"
+
 # Network Policies configuration
 # Allows creating multiple NetworkPolicy resources from a list
 # @ignored