From 228625e4acaa7d74ceca61ac01cf4e1af462040d Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 22 Oct 2026 14:02:20 +0200 Subject: [PATCH 1/4] feat(serving-stack): allow multiple accelerator cpu, nvidia and amd update inferenceClusters, servingStack Signed-off-by: Christopher Haar --- CONTRIBUTING.md | 5 +- apis/aksclusters/definition.yaml | 5 +- apis/civoclusters/definition.yaml | 5 +- apis/eksclusters/definition.yaml | 5 +- apis/gkeclusters/definition.yaml | 5 +- apis/inferenceclasses/definition.yaml | 41 +- apis/inferenceclusters/definition.yaml | 5 +- apis/modeldeployments/definition.yaml | 2 +- apis/nebiusclusters/definition.yaml | 5 +- apis/servingstacks/definition.yaml | 18 + apis/vultrclusters/definition.yaml | 5 +- .../concepts/inference-class-h100-byo.yaml | 2 +- functions/compose-aks-cluster/function/fn.py | 13 + .../compose-aks-cluster/tests/test_fn.py | 80 ++++ functions/compose-civo-cluster/function/fn.py | 16 + .../compose-civo-cluster/tests/test_fn.py | 83 ++++ functions/compose-eks-cluster/function/fn.py | 18 + .../compose-eks-cluster/tests/test_fn.py | 237 ++++++++++ functions/compose-gke-cluster/function/fn.py | 16 + .../compose-gke-cluster/tests/test_fn.py | 102 +++++ .../compose-inference-cluster/function/fn.py | 237 +++++++--- .../tests/test_fn.py | 411 +++++++++++++++-- .../compose-model-deployment/function/cel.py | 4 +- .../compose-model-deployment/function/fn.py | 2 +- .../function/scheduling.py | 22 +- .../compose-model-deployment/tests/test_fn.py | 2 +- .../tests/test_scheduling.py | 200 ++++----- .../function/backends/base.py | 32 +- .../tests/test_backends.py | 276 ++++++++++-- .../compose-model-replica/tests/test_fn.py | 12 +- .../compose-nebius-cluster/function/fn.py | 16 + .../compose-nebius-cluster/tests/test_fn.py | 116 +++++ .../compose-serving-stack/function/fn.py | 16 +- .../function/stacks/__init__.py | 86 +++- .../function/stacks/clouds/civo.py | 18 +- .../function/stacks/clouds/existing.py | 104 ++++- .../stacks/clouds/generated/aicr/aks.py | 16 +- .../stacks/clouds/generated/aicr/eks.py | 15 +- .../stacks/clouds/generated/aicr/gke.py | 16 +- .../function/stacks/clouds/nebius.py | 17 +- .../function/stacks/clouds/vultr.py | 13 +- .../function/stacks/common.py | 80 +++- .../function/stacks/components.py | 13 + functions/compose-serving-stack/generate.py | 104 +++-- .../compose-serving-stack/tests/test_fn.py | 413 ++++++++++++++++-- .../tests/test_stacks.py | 186 ++++++++ .../compose-vultr-cluster/function/fn.py | 23 +- .../compose-vultr-cluster/tests/test_fn.py | 77 +++- schemas/.lock.json | 2 +- .../ai/modelplane/inferenceclass/v1alpha1.py | 24 +- .../modelplane/inferencecluster/v1alpha1.py | 24 +- .../infrastructure/akscluster/v1alpha1.py | 4 +- .../infrastructure/civocluster/v1alpha1.py | 4 +- .../infrastructure/ekscluster/v1alpha1.py | 4 +- .../infrastructure/gkecluster/v1alpha1.py | 4 +- .../infrastructure/nebiuscluster/v1alpha1.py | 4 +- .../infrastructure/servingstack/v1alpha1.py | 10 +- .../infrastructure/vultrcluster/v1alpha1.py | 4 +- .../ai/modelplane/modeldeployment/v1alpha1.py | 2 +- 59 files changed, 2858 insertions(+), 423 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0fdb99f78..6e3ddb2fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -426,7 +426,10 @@ checks to trip. In order: file is high-churn upstream. The fork is deleted the day NVIDIA/aicr#2515 or #2517 lands. 3. Expect `ALLOW`/`DROP` to fail closed on any component the new catalog - adds; classify it with a reason. + adds; classify it with a reason. An `ALLOW` entry also names the DRA + drivers whose device stacks the component belongs to (empty for a + component every cluster needs) - deciding that is part of classifying + it; see design/multi-accelerator.md. 4. Expect the managed-path assertions to fail closed if a values path moved or a `--set` stopped landing; re-derive the path from the new chart before loosening anything. `MODELPLANE_VALUES` fails closed too diff --git a/apis/aksclusters/definition.yaml b/apis/aksclusters/definition.yaml index e8a39647b..3cec68b50 100644 --- a/apis/aksclusters/definition.yaml +++ b/apis/aksclusters/definition.yaml @@ -128,10 +128,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and get the NVIDIA - driver installed by AKS. + driver installed by AKS. CPU pools host CPU inference + workloads: no GPU driver, and the modelplane.ai/cpu + label and taint. enum: - System - GPU + - CPU vmSize: type: string description: >- diff --git a/apis/civoclusters/definition.yaml b/apis/civoclusters/definition.yaml index f45cefffc..a76b1c918 100644 --- a/apis/civoclusters/definition.yaml +++ b/apis/civoclusters/definition.yaml @@ -110,10 +110,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU pools host CPU inference + workloads: a non-GPU size and the modelplane.ai/cpu + label and taint. enum: - System - GPU + - CPU size: type: string description: >- diff --git a/apis/eksclusters/definition.yaml b/apis/eksclusters/definition.yaml index 1f347efa4..8bfa2dddd 100644 --- a/apis/eksclusters/definition.yaml +++ b/apis/eksclusters/definition.yaml @@ -132,10 +132,13 @@ spec: Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and use a - GPU-enabled AMI. + GPU-enabled AMI. CPU groups host CPU inference + workloads: a standard AMI, no GPU, and the + modelplane.ai/cpu label and taint. enum: - System - GPU + - CPU instanceType: type: string description: >- diff --git a/apis/gkeclusters/definition.yaml b/apis/gkeclusters/definition.yaml index d5517f26a..588edc7d0 100644 --- a/apis/gkeclusters/definition.yaml +++ b/apis/gkeclusters/definition.yaml @@ -124,10 +124,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU pools host CPU inference + workloads: a standard node image, no GPU, and the + modelplane.ai/cpu label and taint. enum: - System - GPU + - CPU machineType: type: string description: >- diff --git a/apis/inferenceclasses/definition.yaml b/apis/inferenceclasses/definition.yaml index 44264d65b..6085678fa 100644 --- a/apis/inferenceclasses/definition.yaml +++ b/apis/inferenceclasses/definition.yaml @@ -52,7 +52,7 @@ spec: enum: [GKE, EKS, AKS, Nebius, Vultr, Civo] gke: type: object - required: [machineType, accelerator] + required: [machineType] properties: machineType: type: string @@ -68,7 +68,9 @@ spec: pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in - devices. + devices. Omit for a pool with no GPU, which is + composed with role CPU: a standard node image and the + modelplane.ai/cpu label and taint. required: [type, count] properties: type: @@ -84,7 +86,7 @@ spec: maximum: 16 eks: type: object - required: [instanceType, accelerator] + required: [instanceType] properties: instanceType: type: string @@ -103,7 +105,10 @@ spec: description: >- GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Omit for a + group with no GPU, which is composed with role CPU: a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: @@ -120,7 +125,7 @@ spec: maximum: 16 aks: type: object - required: [vmSize, accelerator] + required: [vmSize] properties: vmSize: type: string @@ -139,7 +144,10 @@ spec: description: >- GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Omit for a + pool with no GPU, which is composed with role CPU: a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: @@ -156,7 +164,7 @@ spec: maximum: 16 nebius: type: object - required: [platform, preset, accelerator] + required: [platform, preset] properties: platform: type: string @@ -197,7 +205,10 @@ spec: description: >- GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Omit for a + group with no GPU, which is composed with role CPU: a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: @@ -214,7 +225,7 @@ spec: maximum: 16 vultr: type: object - required: [plan, accelerator] + required: [plan] properties: plan: type: string @@ -229,7 +240,10 @@ spec: description: >- GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Omit for a + pool with no GPU, which is composed with role CPU: a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: @@ -246,7 +260,7 @@ spec: maximum: 16 civo: type: object - required: [size, accelerator] + required: [size] properties: size: type: string @@ -261,7 +275,10 @@ spec: description: >- GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches - against spec.devices, not this block. + against spec.devices, not this block. Omit for a + pool with no GPU, which is composed with role CPU: a + standard node image and the modelplane.ai/cpu label + and taint. required: [type, count] properties: type: diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index a3b6d897f..bd13e13f5 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -608,10 +608,11 @@ spec: PVCs on this cluster. ModelCache reads this to target the cache PVC. maxLength: 253 - gpuPools: + schedulablePools: type: array description: >- - Schedulable GPU node pools on this cluster, derived from the + Schedulable node pools on this cluster - GPU and CPU alike; + every pool whose class declares devices - derived from the referenced classes and the per-pool node counts. ModelDeployment scheduling matches against these. maxItems: 8 diff --git a/apis/modeldeployments/definition.yaml b/apis/modeldeployments/definition.yaml index 5bb1c5dc4..48531c957 100644 --- a/apis/modeldeployments/definition.yaml +++ b/apis/modeldeployments/definition.yaml @@ -453,7 +453,7 @@ spec: pods: what devices each pod needs from its node. The scheduler matches it against a candidate pool's InferenceClass devices (surfaced on - InferenceCluster status.gpuPools) and places the + InferenceCluster status.schedulablePools) and places the member on a pool that satisfies it, preferring one pool for the whole engine and splitting members across pools only when no pool satisfies diff --git a/apis/nebiusclusters/definition.yaml b/apis/nebiusclusters/definition.yaml index 022f29819..171592e49 100644 --- a/apis/nebiusclusters/definition.yaml +++ b/apis/nebiusclusters/definition.yaml @@ -101,10 +101,13 @@ spec: Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU groups host CPU inference + workloads: a CPU platform, no GPU, and the + modelplane.ai/cpu label and taint. enum: - System - GPU + - CPU platform: type: string description: >- diff --git a/apis/servingstacks/definition.yaml b/apis/servingstacks/definition.yaml index 65722e89a..7d3cdec4b 100644 --- a/apis/servingstacks/definition.yaml +++ b/apis/servingstacks/definition.yaml @@ -63,6 +63,24 @@ spec: - Vultr - Civo - Existing + draDrivers: + type: array + description: >- + DRA driver names whose device stacks this cluster needs + (gpu.nvidia.com, dra.cpu, ...), collected by the cluster + composition from the claim: DRA devices of the cluster's + referenced InferenceClasses. A component tagged with a + driver installs only when that driver is listed; untagged + components always install. When the field is absent no + filtering happens and every component installs, so a + ServingStack from before the field existed keeps its + stack until the cluster composition stamps the list. + maxItems: 16 + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + maxLength: 253 secrets: type: array description: >- diff --git a/apis/vultrclusters/definition.yaml b/apis/vultrclusters/definition.yaml index 6d065cbeb..26346cdee 100644 --- a/apis/vultrclusters/definition.yaml +++ b/apis/vultrclusters/definition.yaml @@ -110,10 +110,13 @@ spec: Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to - exclude non-GPU pods. + exclude non-GPU pods. CPU pools host CPU inference + workloads: a non-GPU plan and the modelplane.ai/cpu + label and taint. enum: - System - GPU + - CPU plan: type: string description: >- diff --git a/docs/manifests/concepts/inference-class-h100-byo.yaml b/docs/manifests/concepts/inference-class-h100-byo.yaml index fc070712d..4c2f32aa3 100644 --- a/docs/manifests/concepts/inference-class-h100-byo.yaml +++ b/docs/manifests/concepts/inference-class-h100-byo.yaml @@ -2,7 +2,7 @@ # # No provisioning block: this class describes hardware that already # exists on a bring-your-own cluster. Modelplane copies the devices block -# onto the cluster's status.gpuPools for the scheduler to match against. +# onto the cluster's status.schedulablePools for the scheduler to match against. apiVersion: modelplane.ai/v1alpha1 kind: InferenceClass metadata: diff --git a/functions/compose-aks-cluster/function/fn.py b/functions/compose-aks-cluster/function/fn.py index a6d52a6a1..8c36d27cf 100644 --- a/functions/compose-aks-cluster/function/fn.py +++ b/functions/compose-aks-cluster/function/fn.py @@ -103,6 +103,13 @@ _GPU_TAINT = "nvidia.com/gpu=true:NoSchedule" _GPU_TAINT_KEY = "nvidia.com/gpu" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" +_CPU_TAINT = "modelplane.ai/cpu=true:NoSchedule" + # Node pool gpuDriver value asking AKS to install the NVIDIA driver on GPU # nodes. The device plugin is not installed in this path - the serving # stack's DRA driver binds GPUs to pods, as on EKS. @@ -410,6 +417,12 @@ def compose_node_pools(self) -> None: _LABEL_POOL: pool.name, } fp.nodeTaints = [_GPU_TAINT] + elif pool.role == "CPU": + fp.nodeLabels = { + _LABEL_POOL: pool.name, + _LABEL_CPU: "true", + } + fp.nodeTaints = [_CPU_TAINT] resource.update( self.rsp.desired.resources[f"nodepool-{pool.name}"], diff --git a/functions/compose-aks-cluster/tests/test_fn.py b/functions/compose-aks-cluster/tests/test_fn.py index 4263875bc..512e2d9da 100644 --- a/functions/compose-aks-cluster/tests/test_fn.py +++ b/functions/compose-aks-cluster/tests/test_fn.py @@ -212,6 +212,41 @@ def _nodepool_gpu(*, cred_kind: str, cred_name: str, zones: list[str] | None, re return fnv1.Resource(resource=resource.dict_to_struct(nodepool), ready=ready) +def _nodepool_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: + """The composed cpupool node pool, with the CPU label and taint and no GPU driver.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "containerservice.azure.m.upbound.io/v1beta1", + "kind": "KubernetesClusterNodePool", + "metadata": {"annotations": {"crossplane.io/external-name": "cpupool"}}, + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "managementPolicies": ["Observe", "Create", "Update", "Delete"], + "initProvider": {"nodeCount": 1}, + "forProvider": { + "kubernetesClusterIdSelector": {"matchControllerRef": True}, + "vnetSubnetIdSelector": {"matchControllerRef": True}, + "mode": "User", + "vmSize": "Standard_D8s_v5", + "osDiskSizeGb": 100, + "orchestratorVersion": "1.34", + "autoScalingEnabled": True, + "minCount": 0, + "maxCount": 4, + "nodeLabels": { + "modelplane.ai/pool": "cpupool", + "modelplane.ai/cpu": "true", + }, + "nodeTaints": ["modelplane.ai/cpu=true:NoSchedule"], + "upgradeSettings": {"maxSurge": "10%"}, + }, + }, + } + ), + ) + + def _provider_config(*, api_version: str) -> fnv1.Resource: """A Ready ProviderConfig that reaches the cluster through its kubeconfig Secret.""" return fnv1.Resource( @@ -293,6 +328,51 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no GPU driver.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-cpupool": _nodepool_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + ), + ), Case( name="NodePoolZones", reason="An AKSCluster passes a pool's zones through to its KubernetesClusterNodePool.", diff --git a/functions/compose-civo-cluster/function/fn.py b/functions/compose-civo-cluster/function/fn.py index 0ba30cade..081c931dc 100644 --- a/functions/compose-civo-cluster/function/fn.py +++ b/functions/compose-civo-cluster/function/fn.py @@ -96,6 +96,12 @@ _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NoSchedule" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Civo's default marketplace applications include the Traefik ingress # controller, which would compete with the serving stack's Envoy Gateway # for LoadBalancer traffic. A "-" prefix removes a default application at @@ -430,6 +436,8 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodepoolv1beta1.NodePool: labels = {_LABEL_POOL: pool.name} if pool.role == "GPU" and pool.gpu: labels[_LABEL_GPU] = pool.gpu.acceleratorType + elif pool.role == "CPU": + labels[_LABEL_CPU] = "true" fp = nodepoolv1beta1.ForProvider( label=pool.name, @@ -451,6 +459,14 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodepoolv1beta1.NodePool: effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + fp.taint = [ + nodepoolv1beta1.TaintItem( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] spec = nodepoolv1beta1.Spec( providerConfigRef=nodepoolv1beta1.ProviderConfigRef( diff --git a/functions/compose-civo-cluster/tests/test_fn.py b/functions/compose-civo-cluster/tests/test_fn.py index c0ae9ae55..92edc33a0 100644 --- a/functions/compose-civo-cluster/tests/test_fn.py +++ b/functions/compose-civo-cluster/tests/test_fn.py @@ -238,6 +238,47 @@ def _node_pool_gpu( ) +def _node_pool_cpu( + *, + management_policies: list[str] | None, + init_node_count: int | None, + node_count: int | None, + ready: fnv1.Ready, +) -> fnv1.Resource: + """The composed NodePool for the cpu-pool pool, with the CPU label and taint and no GPU label.""" + for_provider: dict[str, Any] = { + "label": "cpu-pool", + "size": "g4s.kube.large", + "region": "LON1", + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + "clusterIdSelector": {"matchControllerRef": True}, + "taint": [{"key": "modelplane.ai/cpu", "value": "true", "effect": "NoSchedule"}], + } + if node_count is not None: + for_provider["nodeCount"] = node_count + spec: dict[str, Any] = { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": for_provider, + } + if management_policies is not None: + spec["managementPolicies"] = management_policies + if init_node_count is not None: + spec["initProvider"] = {"nodeCount": init_node_count} + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.civo.m.upbound.io/v1beta1", + "kind": "NodePool", + "spec": spec, + } + ), + ready=ready, + ) + + def _provider_config() -> fnv1.Resource: """The composed provider-helm ProviderConfig for the cluster, which is always ready.""" return fnv1.Resource( @@ -381,6 +422,48 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="ClusterReadyCPUPool", + reason="A CPU-role pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no GPU label.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + size="g4s.kube.large", + maxNodeCount=4, + ), + ], + ), + resources={ + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": _node_pool_cpu( + management_policies=["Observe", "Create", "Update", "Delete"], + init_node_count=1, + node_count=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-helm": _provider_config(), + "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[]), + }, + ), + context=structpb.Struct(), + ), + ), # The autoscaler addresses node groups by the pool ID the provider writes # to the NodePool's external-name annotation once the pool exists. The pool # leaves nodeCount at its default of 1, so this case can't tell a floor diff --git a/functions/compose-eks-cluster/function/fn.py b/functions/compose-eks-cluster/function/fn.py index 95ac2ec2c..72341c41c 100644 --- a/functions/compose-eks-cluster/function/fn.py +++ b/functions/compose-eks-cluster/function/fn.py @@ -226,6 +226,12 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NO_SCHEDULE" +# Label and taint on CPU inference node groups. The label is what the +# serving stack's dra-driver-cpu selects its nodes by; the taint keeps +# system workloads off the group as the GPU taint does on GPU groups, +# and engine pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Kubernetes-API taint effect (as a toleration value, not the EC2 NO_SCHEDULE # form). The EFA DRA driver's DaemonSet must tolerate the GPU taint to run on # the GPU nodes, like the NVIDIA GPU DRA driver does. @@ -910,6 +916,18 @@ def compose_node_groups(self) -> None: effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + fp.labels = { + _LABEL_POOL: pool.name, + _LABEL_CPU: "true", + } + fp.taint = [ + ngv1beta1.TaintItem( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] resource.update( self.rsp.desired.resources[f"nodegroup-{pool.name}"], diff --git a/functions/compose-eks-cluster/tests/test_fn.py b/functions/compose-eks-cluster/tests/test_fn.py index d08a6a7f6..bd16d8e9c 100644 --- a/functions/compose-eks-cluster/tests/test_fn.py +++ b/functions/compose-eks-cluster/tests/test_fn.py @@ -631,6 +631,50 @@ def _nodegroup_gpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: ) +def _nodegroup_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: + """The composed cpu-pool node group, on the standard AMI with the CPU label and taint.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "eks.aws.m.upbound.io/v1beta1", + "kind": "NodeGroup", + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "managementPolicies": ["Observe", "Create", "Update", "Delete"], + "initProvider": {"scalingConfig": {"desiredSize": 1}}, + "forProvider": { + "region": "us-west-2", + "amiType": "AL2023_x86_64_STANDARD", + "instanceTypes": ["m6i.4xlarge"], + "diskSize": 100, + "clusterNameSelector": {"matchControllerRef": True}, + "nodeRoleArnSelector": { + "matchControllerRef": True, + "matchLabels": {"modelplane.ai/iam-role": "node"}, + }, + "subnetIdSelector": { + "matchControllerRef": True, + "matchLabels": {"modelplane.ai/subnet-tier": "private"}, + }, + "scalingConfig": {"minSize": 0, "maxSize": 4}, + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + "taint": [ + { + "key": "modelplane.ai/cpu", + "value": "true", + "effect": "NO_SCHEDULE", + }, + ], + }, + }, + } + ), + ) + + def _launch_template_efa(*, security_groups: list[str] | None) -> fnv1.Resource: """The gpu-h200 EFA launch template, with security_groups on every interface if given.""" # p5en.48xlarge has 16 network cards; one EFA interface per card. @@ -1276,6 +1320,199 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node pool composes its node group on the standard AMI with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + instanceType="m6i.4xlarge", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), + "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), + "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), + "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), + "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-cpu-pool": _nodegroup_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ClusterProviderConfig", cred_name="default" + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + ), + ), # The observed filesystem id lets the function compose the StorageClass # Object, which is ready, against the cluster's own ProviderConfig. The # observed cluster lets it compose the autoscaler Helm release, since diff --git a/functions/compose-gke-cluster/function/fn.py b/functions/compose-gke-cluster/function/fn.py index 18d550b99..e5a48445f 100644 --- a/functions/compose-gke-cluster/function/fn.py +++ b/functions/compose-gke-cluster/function/fn.py @@ -65,6 +65,14 @@ _LABEL_GPU = "modelplane.ai/gpu" _LABEL_POOL = "modelplane.ai/pool" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). GKE auto-taints GPU pools +# with nvidia.com/gpu but knows nothing of CPU inference pools, so the +# taint is set explicitly here. +_LABEL_CPU = "modelplane.ai/cpu" + # GKE's cluster autoscaler uses this label to recognise a node pool as one whose # nodes run the NVIDIA GPU DRA driver, and so models the DRA ResourceSlices such # a node would publish. Without it the autoscaler can't tell that a new node @@ -390,6 +398,14 @@ def compose_node_pools(self) -> None: _LABEL_POOL: pool.name, _LABEL_GPU_DRA_DRIVER: "true", } + elif pool.role == "CPU": + node_config.labels = { + _LABEL_POOL: pool.name, + _LABEL_CPU: "true", + } + node_config.taint = [ + nodepoolv1beta1.TaintItem(key=_LABEL_CPU, value="true", effect="NO_SCHEDULE"), + ] else: node_config.labels = { _LABEL_POOL: pool.name, diff --git a/functions/compose-gke-cluster/tests/test_fn.py b/functions/compose-gke-cluster/tests/test_fn.py index c3ccd534d..fe12aabf4 100644 --- a/functions/compose-gke-cluster/tests/test_fn.py +++ b/functions/compose-gke-cluster/tests/test_fn.py @@ -279,6 +279,46 @@ def _nodepool_gpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: ) +def _nodepool_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: + """The composed NodePool for the XR's cpu-pool.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "container.gcp.m.upbound.io/v1beta1", + "kind": "NodePool", + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "forProvider": { + "location": "us-central1", + "clusterSelector": {"matchControllerRef": True}, + "initialNodeCount": 1, + "autoscaling": {"minNodeCount": 0, "maxNodeCount": 8}, + "nodeConfig": { + "machineType": "e2-standard-8", + "diskSizeGb": 100, + "imageType": "COS_CONTAINERD", + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform", + ], + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + "taint": [ + { + "key": "modelplane.ai/cpu", + "value": "true", + "effect": "NO_SCHEDULE", + }, + ], + }, + }, + }, + } + ), + ) + + def _service_account(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed GCP ServiceAccount.""" return fnv1.Resource( @@ -421,6 +461,68 @@ def _to_dict(msg: message.Message) -> dict: ), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no guest accelerator.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + machineType="e2-standard-8", + ), + ], + ), + ), + required_resources={ + "gcp-provider-config": fnv1.Resources( + items=[_gcp_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "network": _network( + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "projectservice-filestore": _projectservice_filestore( + cred_kind="ClusterProviderConfig", + cred_name="default", + ), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default"), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodepool-system": _nodepool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodepool-cpu-pool": _nodepool_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), + "service-account": _service_account( + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "service-account-key": _service_account_key(cred_kind="ClusterProviderConfig", cred_name="default"), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "gcp-provider-config": fnv1.ResourceSelector( + api_version="gcp.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), # An XR with no composed resources would aggregate to trivially ready, so # the function marks it not ready. Case( diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index e5d1dd4ee..74b5599f2 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -19,11 +19,12 @@ Existing) to determine how the cluster is obtained, then composes a ServingStack on it. -GPU node pools reference InferenceClasses. For provisioned clusters -the class's provisioning block describes how to build the pool; -for BYO (Existing) clusters the class is a pure description of pools -that already exist. Either way, the class's resources block populates -status.gpuPools so the scheduler can match models. +Node pools reference InferenceClasses. For provisioned clusters +the class's provisioning block describes how to build the pool (a GPU +pool when it names an accelerator, a CPU pool when it doesn't); for BYO +(Existing) clusters the class is a pure description of pools that +already exist. Either way, the class's devices populate +status.schedulablePools so the scheduler can match models. For provisioned clusters, a system node pool is injected automatically to host control-plane components (Envoy Gateway, Prometheus, etc.). @@ -81,6 +82,29 @@ CONDITION_REASON_BACKEND_HEALTHY = "BackendHealthy" CONDITION_REASON_INSTALLING = "Installing" CONDITION_REASON_INVALID_NODE_POOL = "InvalidNodePool" +CONDITION_REASON_UNSUPPORTED_DEVICES = "UnsupportedDevices" + +# The DRA drivers each cloud's serving stack supports. A hand-kept +# mirror of supported_drivers() in compose-serving-stack's stacks +# package - functions are self-contained by design, so keep the two in +# sync by hand. A stale mirror fails closed: it rejects a newly +# supported driver until someone updates it, and never admits an +# unsupported one, because the serving stack join's raise backstops +# that direction. EKS's dra.net arrives outside the serving stack +# (compose_eks_cluster installs the EFA DRA driver when a pool asks for +# the fabric). Existing is deliberately absent and never gates: the +# user owns a BYO cluster and may have installed any driver on it, so a +# claimed driver beyond the stack's own installs nothing and is not +# rejected - a typo there surfaces when its ResourceClaim can't +# allocate, not as a condition. +_SUPPORTED_DRA_DRIVERS: dict[str, frozenset[str]] = { + CLUSTER_SOURCE_GKE: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_EKS: frozenset({"gpu.nvidia.com", "dra.cpu", "dra.net"}), + CLUSTER_SOURCE_AKS: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_NEBIUS: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_VULTR: frozenset({"gpu.nvidia.com", "dra.cpu"}), + CLUSTER_SOURCE_CIVO: frozenset({"gpu.nvidia.com", "dra.cpu"}), +} # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" @@ -301,6 +325,9 @@ def compose(self) -> None: return source = cluster.source + if not self.check_devices_supported(source): + return + if source == CLUSTER_SOURCE_GKE: self.compose_gke(cluster.gke) elif source == CLUSTER_SOURCE_EKS: @@ -571,6 +598,61 @@ def resolve_classes(self) -> bool: return True + def dra_drivers(self) -> list[str]: + """DRA driver names claimed across the referenced classes' devices. + + Collected verbatim onto ServingStack spec.draDrivers, which filters + the cloud's tagged components to the device stacks the cluster's + classes actually claim. Only claim: DRA devices count: a Synthetic + device is described for scheduling and never claimed, and its driver + is only the attribute domain its CEL selectors read, not an + installable stack. Sorted, so the ServingStack spec is stable across + reconciles. + """ + drivers: set[str] = set() + for cls in self.classes.values(): + for device in cls.spec.devices or []: + if (device.claim or "DRA") == "DRA": + drivers.add(device.driver) + return sorted(drivers) + + def check_devices_supported(self, source: Cloud) -> bool: + """Gate on a class claiming a driver the cloud's stack can't serve. + + Checked here, before any cluster or ServingStack is composed, because + the cluster would provision devices nothing can drive. A typo'd or + not-yet-supported driver lands here too; unknown and unsupported are + the same failure. Existing never gates (see _SUPPORTED_DRA_DRIVERS). + """ + supported = _SUPPORTED_DRA_DRIVERS.get(source) + if supported is None: + return True + for name in sorted(self.classes): + cls = self.classes[name] + claimed = {d.driver for d in (cls.spec.devices or []) if (d.claim or "DRA") == "DRA"} + unsupported = claimed - supported + if not unsupported: + continue + msg = ( + f"class {name} claims {', '.join(sorted(unsupported))}; " + f"the {source} serving stack installs {', '.join(sorted(supported))}" + ) + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_UNSUPPORTED_DEVICES, + message=msg, + ), + ) + response.warning(self.rsp, msg) + # Only the guard and namespaces, both marked ready, are composed + # when gating, so the XR would otherwise be ready. + self.rsp.desired.composite.ready = fnv1.READY_FALSE + return False + return True + def compose_gke(self, gke: v1alpha1.Gke | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned GKE cluster. Composes the GKECluster XR, waits for it to be ready, @@ -608,7 +690,7 @@ def compose_gke(self, gke: v1alpha1.Gke | None) -> None: if not backend_exists: response.normal(self.rsp, "GKE cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=gke_ready) def compose_eks(self, eks: v1alpha1.Eks | None) -> None: @@ -649,7 +731,7 @@ def compose_eks(self, eks: v1alpha1.Eks | None) -> None: if not backend_exists: response.normal(self.rsp, "EKS cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=eks_ready) def compose_aks(self, aks: v1alpha1.Aks | None) -> None: @@ -688,7 +770,7 @@ def compose_aks(self, aks: v1alpha1.Aks | None) -> None: if not backend_exists: response.normal(self.rsp, "AKS cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=aks_ready) def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: @@ -737,7 +819,7 @@ def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: if not backend_exists: response.normal(self.rsp, "Nebius cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=nebius_ready) def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: @@ -776,7 +858,7 @@ def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: if not backend_exists: response.normal(self.rsp, "Vultr cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=vultr_ready) def compose_civo(self, civo: v1alpha1.Civo | None) -> None: @@ -821,7 +903,7 @@ def compose_civo(self, civo: v1alpha1.Civo | None) -> None: if not backend_exists: response.normal(self.rsp, "Civo cluster ready, composing backend") - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=civo_ready) def compose_existing(self, existing: v1alpha1.Existing | None) -> None: @@ -850,7 +932,7 @@ def compose_existing(self, existing: v1alpha1.Existing | None) -> None: ) self.compose_serving_stack(backend_secrets, CLUSTER_SOURCE_EXISTING) - self.write_status(self.gpu_pools()) + self.write_status(self.schedulable_pools()) self.derive_conditions(cluster_ready=True) def compose_serving_stack( @@ -865,9 +947,12 @@ def compose_serving_stack( cloud names the cluster's source (this XR's spec.cluster.source) and selects the component list the serving stack installs, including cloud specifics like where the node image puts the - NVIDIA driver. gpu carries per-pool driver configuration the - component list can't know at build time (see - civo_nvlink_disabled_pools). + NVIDIA driver. spec.draDrivers is always set - an empty list when + no class claims a device - so the stack filters its tagged device + stacks to the drivers the classes claim; only a ServingStack from + before the field existed goes unfiltered. gpu carries per-pool + driver configuration the component list can't know at build time + (see civo_nvlink_disabled_pools). """ # The gateway's name and the CAs it should accept client certificates # from. The name is Modelplane's own, derived from this cluster's name; @@ -884,6 +969,7 @@ def compose_serving_stack( stack=self.xr.spec.stack, cloud=cloud, gateway=gateway, + draDrivers=[ssv1alpha1.DraDriver(d) for d in self.dra_drivers()], ) if gpu is not None: spec.gpu = gpu @@ -945,14 +1031,16 @@ def compose_cluster_provider_config( ) self.rsp.desired.resources["cluster-provider-config-kubernetes"].ready = fnv1.READY_TRUE - def write_status(self, gpu_pools: list[dict[str, object]]) -> None: + def write_status(self, schedulable_pools: list[dict[str, object]]) -> None: """Write the InferenceCluster status.""" status = v1alpha1.Status( providerConfigRef=v1alpha1.ProviderConfigRef( name=resource.child_name(_name(self.xr.metadata), "cluster-kubeconfig"), ), namespace=_NAMESPACE_SYSTEM, - gpuPools=gpu_pools, # gpu_pools builds the by-alias wire form; pydantic coerces it to GpuPool + # schedulable_pools builds the by-alias wire form; pydantic + # coerces it to SchedulablePool. + schedulablePools=schedulable_pools, ) cache_storage_class = self.observed_cache_storage_class() if cache_storage_class: @@ -1047,22 +1135,30 @@ def compose_gke_cluster(self, gke: v1alpha1.Gke) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.gke - gke_node_pools.append( - gkev1alpha1.NodePool( - name=pool.name, - role="GPU", - machineType=prov.machineType, - diskSizeGb=prov.diskSizeGb, - nodeCount=pool.nodeCount, - minNodeCount=pool.minNodeCount, - maxNodeCount=pool.maxNodeCount, - gpu=gkev1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - acceleratorCount=prov.accelerator.count, - ), - zones=[gkev1alpha1.Zone(z) for z in pool.zones or []], - ) + # A class with no accelerator describes a CPU pool: a standard + # node image, no GPU attachment, and the modelplane.ai/cpu + # label and taint composed by compose-gke-cluster. + node_pool = gkev1alpha1.NodePool( + name=pool.name, + role="GPU" if prov.accelerator else "CPU", + machineType=prov.machineType, + diskSizeGb=prov.diskSizeGb, + nodeCount=pool.nodeCount, + minNodeCount=pool.minNodeCount, + maxNodeCount=pool.maxNodeCount, ) + if prov.accelerator: + node_pool.gpu = gkev1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + acceleratorCount=prov.accelerator.count, + ) + # Only set zones when the pool names some: GKE requires a + # non-empty list when zones are set, and resource.update + # serializes with exclude_unset, so leaving it unset keeps an + # empty list out of the GKECluster spec. + if pool.zones: + node_pool.zones = [gkev1alpha1.Zone(z) for z in pool.zones] + gke_node_pools.append(node_pool) gke_spec = gkev1alpha1.Spec( region=gke.region, @@ -1110,19 +1206,26 @@ def compose_eks_cluster(self, eks: v1alpha1.Eks) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.eks + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = eksv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", instanceType=prov.instanceType, diskSizeGb=prov.diskSizeGb, nodeCount=pool.nodeCount, minNodeCount=pool.minNodeCount, maxNodeCount=pool.maxNodeCount, - gpu=eksv1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), - zones=[eksv1alpha1.Zone(z) for z in pool.zones or []], ) + if prov.accelerator: + node_pool.gpu = eksv1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) + # Only set zones when the pool names some, so an unzoned pool's + # EKSCluster spec stays free of zones: [] (resource.update + # serializes with exclude_unset). + if pool.zones: + node_pool.zones = [eksv1alpha1.Zone(z) for z in pool.zones] # Only set capacityBlock when the pool has one. resource.update # serializes with exclude_unset, so leaving it unset keeps it out # of the EKSCluster spec rather than emitting capacityBlock: null. @@ -1182,18 +1285,21 @@ def compose_aks_cluster(self, aks: v1alpha1.Aks) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.aks + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = aksv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", vmSize=prov.vmSize, diskSizeGb=prov.diskSizeGb, nodeCount=pool.nodeCount, minNodeCount=pool.minNodeCount, maxNodeCount=pool.maxNodeCount, - gpu=aksv1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), ) + if prov.accelerator: + node_pool.gpu = aksv1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) # Only set zones when the pool names some: the AKSCluster XRD # requires a non-empty list, and many GPU VM sizes aren't zonal. if pool.zones: @@ -1253,18 +1359,22 @@ def compose_nebius_cluster(self, nebius: v1alpha1.Nebius) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.nebius + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). driversPreset rides the gpu block: a CPU + # platform preinstalls no NVIDIA stack. node_pool = nebiusv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", platform=prov.platform, preset=prov.preset, diskSizeGb=prov.diskSizeGb, nodeCount=pool.nodeCount, - gpu=nebiusv1alpha1.Gpu( + ) + if prov.accelerator: + node_pool.gpu = nebiusv1alpha1.Gpu( acceleratorType=prov.accelerator.type, driversPreset=prov.driversPreset, - ), - ) + ) # mk8s node groups are either fixed size or autoscaled, never # both. Only set the autoscaling bounds when the pool opts in, so # fixed-size pools stay fixed (resource.update serializes with @@ -1410,15 +1520,18 @@ def compose_vultr_cluster(self, vultr: v1alpha1.Vultr) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.vultr + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = vultrv1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", plan=prov.plan, nodeCount=pool.nodeCount, - gpu=vultrv1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), ) + if prov.accelerator: + node_pool.gpu = vultrv1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) # Only set the autoscaling bounds when the pool opts in, so # fixed-size pools stay fixed (resource.update serializes with # exclude_unset, keeping unset bounds out of the VultrCluster @@ -1528,7 +1641,7 @@ def civo_nvlink_disabled_pools(self) -> list[str]: if not cls or not cls.spec.provisioning or not cls.spec.provisioning.civo: continue accelerator = cls.spec.provisioning.civo.accelerator - if accelerator.type == _ACCELERATOR_H100 and accelerator.count == 1: + if accelerator and accelerator.type == _ACCELERATOR_H100 and accelerator.count == 1: pools.append(pool.name) return pools @@ -1557,15 +1670,18 @@ def compose_civo_cluster(self, civo: v1alpha1.Civo) -> None: response.warning(self.rsp, msg) return prov = cls.spec.provisioning.civo + # A class with no accelerator describes a CPU pool (see the GKE + # builder above). node_pool = civov1alpha1.NodePool( name=pool.name, - role="GPU", + role="GPU" if prov.accelerator else "CPU", size=prov.size, nodeCount=pool.nodeCount, - gpu=civov1alpha1.Gpu( - acceleratorType=prov.accelerator.type, - ), ) + if prov.accelerator: + node_pool.gpu = civov1alpha1.Gpu( + acceleratorType=prov.accelerator.type, + ) # Only set the autoscaling bounds when the pool opts in, so # fixed-size pools stay fixed (resource.update serializes with # exclude_unset, keeping unset bounds out of the CivoCluster @@ -1820,15 +1936,16 @@ def resolve_gke_backend_secrets(self, *, gke_ready: bool, backend_exists: bool) return None - def gpu_pools(self) -> list[dict[str, object]]: - """Derive status.gpuPools from each node pool's class. + def schedulable_pools(self) -> list[dict[str, object]]: + """Derive status.schedulablePools from each node pool's class. The class declares the node's devices (DRA-style); the pool declares how many nodes. We copy the class's devices verbatim so ModelDeployment.nodeSelector can match against them, and record the node - count for the scheduler's available-node gate. + count for the scheduler's available-node gate. Nothing here is specific + to GPUs: a CPU pool's dra.cpu device lists like any other. """ - gpu_pools = [] + pools = [] for pool in self.xr.spec.nodePools or []: cls = self.classes.get(pool.className) if not cls or not cls.spec.devices: @@ -1839,14 +1956,14 @@ def gpu_pools(self) -> list[dict[str, object]]: # names (bool/int) rather than the generated bool_/int_ attributes, # so the published status matches the InferenceClass schema. devices = [d.model_dump(by_alias=True, exclude_none=True) for d in cls.spec.devices] - gpu_pools.append( + pools.append( { "name": pool.name, "nodes": pool.maxNodeCount or pool.nodeCount, "devices": devices, } ) - return gpu_pools + return pools def observed_gke_secrets(self) -> list[gkev1alpha1.Secret] | None: """Read the GKECluster's status.secrets from observed state.""" diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index dfbcb7600..2425a9082 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -61,12 +61,14 @@ def _inference_cluster(*, cluster: v1alpha1.Cluster, node_pools: list[v1alpha1.N ) -def _desired_inference_cluster(*, gpu_pools: list[dict], cache: dict | None, gateway: dict | None) -> fnv1.Resource: +def _desired_inference_cluster( + *, schedulable_pools: list[dict], cache: dict | None, gateway: dict | None +) -> fnv1.Resource: """The desired InferenceCluster XR's status, with its cache and gateway unless they're None.""" status: dict = { "providerConfigRef": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, "namespace": "modelplane-system", - "gpuPools": gpu_pools, + "schedulablePools": schedulable_pools, } if cache is not None: status["cache"] = cache @@ -101,6 +103,32 @@ def _inference_class(*, name: str, count: int, memory: str, provisioning: dict) ) +def _cpu_inference_class(*, name: str, cpus: str, provisioning: dict) -> fnv1.Resource: + """An InferenceClass of one DRA-claimed CPU device of cpus cores, as its class requirement returns it.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "InferenceClass", + "metadata": {"name": name}, + "spec": { + "devices": [ + { + "name": "cpu", + "claim": "DRA", + "driver": "dra.cpu", + "deviceClassName": "dra.cpu", + "count": 1, + "capacity": {"cpu": {"value": cpus}}, + }, + ], + "provisioning": provisioning, + }, + } + ) + ) + + def _inference_gateway(*, name: str, cluster: str, status: dict | None) -> fnv1.Resource: """An InferenceGateway on cluster, as the gateways requirement returns it, with status unless it's None.""" gateway: dict = { @@ -267,6 +295,33 @@ def _vultr_cluster(*, credentials: dict | None, ready: fnv1.Ready) -> fnv1.Resou ) +def _vultr_cluster_cpu(*, ready: fnv1.Ready) -> fnv1.Resource: + """The composed VultrCluster with cpu-pool, a CPU-role pool with no gpu block.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "VultrCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "ewr", + "kubernetesVersion": "v1.36.2+1", + "nodePools": [ + { + "name": "cpu-pool", + "role": "CPU", + "plan": "vc2-6c-16gb", + "nodeCount": 2, + "maxNodeCount": 4, + }, + ], + }, + } + ), + ready=ready, + ) + + def _civo_cluster(*, node_pools: list[dict], credentials: dict | None, ready: fnv1.Ready) -> fnv1.Resource: """The composed CivoCluster in LON1 with node_pools, using credentials unless they're None.""" spec: dict = {"region": "LON1", "nodePools": node_pools} @@ -309,13 +364,29 @@ def _cluster_provider_config(*, kubeconfig: str, identity: dict | None) -> fnv1. def _serving_stack( - *, cloud: str, secrets: list[dict], client_cas: list[dict] | None, gpu: dict | None, ready: fnv1.Ready + *, + cloud: str, + secrets: list[dict], + client_cas: list[dict] | None, + gpu: dict | None, + ready: fnv1.Ready, + dra_drivers: tuple[str, ...], ) -> fnv1.Resource: - """The composed ServingStack, its gateway accepting client_cas and its spec.gpu set to gpu unless they're None.""" + """The composed ServingStack, its gateway accepting client_cas and its spec.gpu set to gpu unless they're None. + + dra_drivers is the drivers the cluster's classes claim, always stamped + on the spec. + """ gateway: dict = {"hostname": "gateway-test-cluster-09532.modelplane-system.svc.cluster.local"} if client_cas is not None: gateway["clientCAs"] = client_cas - spec: dict = {"cloud": cloud, "gateway": gateway, "stack": "Standard", "secrets": secrets} + spec: dict = { + "cloud": cloud, + "gateway": gateway, + "stack": "Standard", + "secrets": secrets, + "draDrivers": list(dra_drivers), + } if gpu is not None: spec["gpu"] = gpu return fnv1.Resource( @@ -469,7 +540,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -494,6 +565,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -573,7 +645,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -607,6 +679,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}, { @@ -703,7 +776,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -827,7 +900,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -933,7 +1006,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -958,6 +1031,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -1061,7 +1135,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1234,7 +1308,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1409,7 +1483,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1457,6 +1531,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="GKE", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, { @@ -1602,7 +1677,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1657,6 +1732,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="EKS", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -1767,7 +1843,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -1921,7 +1997,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2059,7 +2135,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100-pool", "nodes": 4, @@ -2247,7 +2323,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100-pool", "nodes": 4, @@ -2318,6 +2394,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Nebius", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, { @@ -2418,7 +2495,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100pool", "nodes": 4, @@ -2596,7 +2673,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100pool", "nodes": 4, @@ -2658,6 +2735,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="AKS", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -2753,7 +2831,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2897,7 +2975,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -2945,6 +3023,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="GKE", + dra_drivers=("gpu.nvidia.com",), secrets=[ {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, { @@ -3029,7 +3108,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -3128,7 +3207,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -3263,7 +3342,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -3293,6 +3372,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Vultr", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -3328,6 +3408,148 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # A class with no accelerator describes a CPU pool: the VultrCluster pool is + # CPU-role with no gpu block, the ServingStack installs the dra.cpu stack, + # and the class's CPU device lists in schedulablePools like any other. + ComposeCase( + name="VultrReadyCPUPool", + reason="Once its VultrCluster is Ready, a Vultr cluster whose pool's class names no accelerator composes a CPU-role pool, a ServingStack claiming dra.cpu and the Usage.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="Vultr", vultr=v1alpha1.Vultr(region="ewr")), + node_pools=[ + v1alpha1.NodePool(name="cpu-pool", className="cpu-vultr", nodeCount=2, maxNodeCount=4), + ], + ), + resources={ + "vultr-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "VultrCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "ewr", + "nodePools": [ + { + "name": "cpu-pool", + "role": "CPU", + "plan": "vc2-6c-16gb", + "nodeCount": 2, + }, + ], + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + }, + } + ), + ), + "activation": _observed_activation_policy( + activated=["kubernetes.vke.vultr.m.upbound.io", "kubernetesnodepools.vke.vultr.m.upbound.io"] + ), + }, + ), + required_resources={ + "class-cpu-vultr": fnv1.Resources( + items=[ + _cpu_inference_class( + name="cpu-vultr", + cpus="6", + provisioning={ + "provider": "Vultr", + "vultr": {"plan": "vc2-6c-16gb"}, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + schedulable_pools=[ + { + "name": "cpu-pool", + "nodes": 4, + "devices": [ + { + "name": "cpu", + "claim": "DRA", + "driver": "dra.cpu", + "deviceClassName": "dra.cpu", + "count": 1, + "capacity": {"cpu": {"value": "6"}}, + }, + ], + }, + ], + cache=None, + gateway=None, + ), + resources={ + "activation": _activation_policy( + activate=["kubernetes.vke.vultr.m.upbound.io", "kubernetesnodepools.vke.vultr.m.upbound.io"], + ready=fnv1.READY_TRUE, + ), + "vultr-cluster": _vultr_cluster_cpu(ready=fnv1.READY_TRUE), + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="test-cluster-kubeconfig-abcde", identity=None + ), + "serving-stack": _serving_stack( + cloud="Vultr", + secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + dra_drivers=("dra.cpu",), + ), + "usage-vultr-by-backend": _backend_usage(cluster_kind="VultrCluster"), + }, + ), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="Vultr cluster ready, composing backend")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-cpu-vultr": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="cpu-vultr" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + ], + ), + ), # minNodeCount stays unset so the pool's autoscaling floor defaults to its # node count downstream. ComposeCase( @@ -3375,7 +3597,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -3497,7 +3719,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -3653,7 +3875,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l40s-pool", "nodes": 4, @@ -3701,6 +3923,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Civo", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -3841,7 +4064,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "h100-pool", "nodes": 1, @@ -3888,6 +4111,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Civo", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu={"pools": [{"name": "h100-pool", "disableNvLink": True}]}, @@ -4037,7 +4261,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -4069,6 +4293,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4160,7 +4385,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -4187,6 +4412,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4264,7 +4490,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -4291,6 +4517,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4369,7 +4596,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -4395,6 +4622,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4478,7 +4706,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[ + schedulable_pools=[ { "name": "l4-pool", "nodes": 4, @@ -4503,6 +4731,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=("gpu.nvidia.com",), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4661,7 +4890,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[], + schedulable_pools=[], cache=None, gateway={ "address": "34.55.100.10", @@ -4676,6 +4905,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[{"name": "fleet-0", "certificate": "fleet-ca"}], gpu=None, @@ -4734,7 +4964,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[], cache=None, gateway={"address": "34.55.100.10", "caCertificate": "cluster-ca"} + schedulable_pools=[], cache=None, gateway={"address": "34.55.100.10", "caCertificate": "cluster-ca"} ), resources={ "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -4742,6 +4972,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=None, gpu=None, @@ -4804,7 +5035,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_inference_cluster(gpu_pools=[], cache=None, gateway={"address": "34.55.100.10"}), + composite=_desired_inference_cluster( + schedulable_pools=[], cache=None, gateway={"address": "34.55.100.10"} + ), resources={ "usage-replicas": _guard_cluster_usage(reason="InferenceGateways use this InferenceCluster"), "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -4812,6 +5045,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[{"name": "fleet-0", "certificate": "fleet-ca"}], gpu=None, @@ -4873,7 +5107,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_inference_cluster(gpu_pools=[], cache=None, gateway=None), + composite=_desired_inference_cluster(schedulable_pools=[], cache=None, gateway=None), resources={ "usage-replicas": _guard_cluster_usage(reason="InferenceGateways use this InferenceCluster"), "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -4881,6 +5115,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[{"name": "fleet-0", "certificate": "fleet-ca"}], gpu=None, @@ -4952,7 +5187,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - gpu_pools=[], + schedulable_pools=[], cache=None, gateway={ "address": "34.55.100.10", @@ -4967,6 +5202,7 @@ def _to_dict(msg: message.Message) -> dict: ), "serving-stack": _serving_stack( cloud="Existing", + dra_drivers=(), secrets=[{"type": "Kubeconfig", "name": "my-kubeconfig", "key": "kubeconfig"}], client_cas=[ {"name": "aaa", "certificate": "aaa-ca"}, @@ -5042,3 +5278,100 @@ def test_compose(case: ComposeCase) -> None: def test_gateway_hostname(case: GatewayHostnameCase) -> None: """_gateway_hostname derives a cluster's gateway hostname.""" assert fn._gateway_hostname(case.cluster_name) == case.want, case.reason + + +def _composer(*, source: fn.Cloud, devices_by_class: dict[str, list[dict]]) -> fn.Composer: + """A Composer for a test-cluster of the given source whose resolved + classes each declare the given devices, calling the derivation and + gate directly - through RunFunction every combination would need a + whole compose case.""" + cluster = v1alpha1.Cluster(source=source) + req = fnv1.RunFunctionRequest( + observed=fnv1.State(composite=_inference_cluster(cluster=cluster, node_pools=None)), + ) + composer = fn.Composer(req, fnv1.RunFunctionResponse()) + for name, devices in devices_by_class.items(): + composer.classes[name] = fn.iclv1alpha1.InferenceClass.model_validate( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "InferenceClass", + "metadata": {"name": name}, + "spec": {"devices": devices}, + } + ) + return composer + + +def test_dra_drivers_sorted() -> None: + """dra_drivers collects each claim: DRA device's driver once, sorted.""" + composer = _composer( + source="Existing", + devices_by_class={ + "nvidia-a": [{"name": "gpu", "claim": "DRA", "driver": "gpu.nvidia.com", "deviceClassName": "g"}], + "nvidia-b": [{"name": "gpu", "claim": "DRA", "driver": "gpu.nvidia.com", "deviceClassName": "g"}], + "cpu": [{"name": "cpu", "claim": "DRA", "driver": "dra.cpu", "deviceClassName": "dra.cpu"}], + }, + ) + assert composer.dra_drivers() == ["dra.cpu", "gpu.nvidia.com"] + + +def test_dra_drivers_synthetic() -> None: + """A Synthetic device is never claimed: its driver names the attribute + domain its CEL selectors read, not an installable stack, so it stays + off spec.draDrivers. An unset claim defaults to DRA and counts.""" + composer = _composer( + source="Existing", + devices_by_class={ + "efa": [ + {"name": "gpu", "driver": "gpu.nvidia.com", "deviceClassName": "g"}, + {"name": "fabric", "claim": "Synthetic", "driver": "fabric.example.com"}, + ], + }, + ) + assert composer.dra_drivers() == ["gpu.nvidia.com"] + + +def test_unsupported_devices() -> None: + """A class claiming a driver its cloud can't serve gates ClusterReady + with UnsupportedDevices and the design's message, before any cluster + or ServingStack is composed.""" + composer = _composer( + source="Vultr", + devices_by_class={ + "amd-mi355x-8x": [ + {"name": "gpu", "claim": "DRA", "driver": "gpu.amd.com", "deviceClassName": "gpu.amd.com"}, + ], + }, + ) + assert composer.check_devices_supported("Vultr") is False + conditions = {c.type: c for c in composer.rsp.conditions} + assert conditions["ClusterReady"].status == fnv1.STATUS_CONDITION_FALSE + assert conditions["ClusterReady"].reason == "UnsupportedDevices" + assert conditions["ClusterReady"].message == ( + "class amd-mi355x-8x claims gpu.amd.com; the Vultr serving stack installs dra.cpu, gpu.nvidia.com" + ) + assert composer.rsp.desired.composite.ready == fnv1.READY_FALSE + + +def test_external_driver() -> None: + """dra.net is declared external on EKS - compose_eks_cluster installs + the EFA driver - so it passes there and gates everywhere else.""" + devices = { + "efa": [{"name": "efa", "claim": "DRA", "driver": "dra.net", "deviceClassName": "efa.networking.k8s.aws"}], + } + assert _composer(source="EKS", devices_by_class=devices).check_devices_supported("EKS") is True + assert _composer(source="GKE", devices_by_class=devices).check_devices_supported("GKE") is False + + +def test_existing_never_gates() -> None: + """Existing is BYO: any driver is treated as externally installed, so + nothing gates - a typo'd driver surfaces when its ResourceClaim can't + allocate, not as a condition.""" + composer = _composer( + source="Existing", + devices_by_class={ + "byo": [{"name": "gpu", "claim": "DRA", "driver": "gpu.example.com", "deviceClassName": "g"}], + }, + ) + assert composer.check_devices_supported("Existing") is True + assert list(composer.rsp.conditions) == [] diff --git a/functions/compose-model-deployment/function/cel.py b/functions/compose-model-deployment/function/cel.py index a4efea60f..d8aafe335 100644 --- a/functions/compose-model-deployment/function/cel.py +++ b/functions/compose-model-deployment/function/cel.py @@ -181,8 +181,8 @@ def __init__(self, expr: str) -> None: def matches(self, device: dict) -> bool: """Evaluate the selector against one device. - `device` is the raw dict from a pool's status.gpuPools devices - entry, shaped like: + `device` is the raw dict from a pool's status.schedulablePools + devices entry, shaped like: device = { "driver": "", diff --git a/functions/compose-model-deployment/function/fn.py b/functions/compose-model-deployment/function/fn.py index 66a3c5a53..440cc9e01 100644 --- a/functions/compose-model-deployment/function/fn.py +++ b/functions/compose-model-deployment/function/fn.py @@ -170,7 +170,7 @@ def _inference_cluster( ic.status = ic.status or icv1alpha1.Status() ic.status.providerConfigRef = ic.status.providerConfigRef or icv1alpha1.ProviderConfigRef() ic.status.gateway = ic.status.gateway or icv1alpha1.Gateway() - ic.status.gpuPools = ic.status.gpuPools or [] + ic.status.schedulablePools = ic.status.schedulablePools or [] return ic diff --git a/functions/compose-model-deployment/function/scheduling.py b/functions/compose-model-deployment/function/scheduling.py index c46388b2d..4396049c3 100644 --- a/functions/compose-model-deployment/function/scheduling.py +++ b/functions/compose-model-deployment/function/scheduling.py @@ -123,7 +123,7 @@ def _published_count(value: int | None) -> int: Cluster status counts (a pool's node count, a device's per-node count) are unconstrained on the wire: the InferenceCluster XRD puts no minimum on - status.gpuPools[].nodes or [].devices[].count, so a published value can be 0 + status.schedulablePools[].nodes or [].devices[].count, so a published value can be 0 (an autoscaled-to-zero pool's node count), None, or - in a malformed status - negative. All must read as "none available". The load-bearing case is a device count of 0: it differs from a deployment-side count, which the XRD @@ -380,7 +380,7 @@ def _device_satisfies(device: icv1alpha1.Device, programs: list[cel.Program]) -> return all(p.matches(raw) for p in programs) -def _match_member(pool: icv1alpha1.GpuPool, member: _CompiledMember) -> list[DeviceRequest] | None: +def _match_member(pool: icv1alpha1.SchedulablePool, member: _CompiledMember) -> list[DeviceRequest] | None: """Match a member's requests against a pool. Returns the resolved claim: DRA DeviceRequests when the pool satisfies @@ -452,16 +452,16 @@ def _member_cost(member: _CompiledMember, resolved: list[DeviceRequest]) -> int: return member.nodes if resolved else 0 -def _gpu_pools(cluster: icv1alpha1.InferenceCluster) -> list[icv1alpha1.GpuPool]: - """The cluster's published GPU pools, empty if it has none yet.""" +def _schedulable_pools(cluster: icv1alpha1.InferenceCluster) -> list[icv1alpha1.SchedulablePool]: + """The cluster's published schedulable pools, empty if it has none yet.""" if not cluster.status: return [] - return cluster.status.gpuPools or [] + return cluster.status.schedulablePools or [] -def _pool_by_name(cluster: icv1alpha1.InferenceCluster, pool_name: str) -> icv1alpha1.GpuPool | None: +def _pool_by_name(cluster: icv1alpha1.InferenceCluster, pool_name: str) -> icv1alpha1.SchedulablePool | None: """The cluster's published pool with this name, or None.""" - for pool in _gpu_pools(cluster): + for pool in _schedulable_pools(cluster): if (pool.name or "") == pool_name: return pool return None @@ -557,7 +557,7 @@ def _build_ledger( free: dict[tuple[str, str], int] = {} for cluster in clusters: name = _name(cluster.metadata) - for pool in _gpu_pools(cluster): + for pool in _schedulable_pools(cluster): free[(name, pool.name or "")] = _published_count(pool.nodes) def charge(cluster_name: str, pool_name: str, nodes: int) -> None: @@ -766,7 +766,7 @@ def _place_engines( # member so a later member sees capacity an earlier one took. Discarded # wholesale when any member fails to place, so partial placement never # leaks into the real ledger. - trial = {(pool.name or ""): ledger.available(cluster_name, pool.name or "") for pool in _gpu_pools(cluster)} + trial = {(pool.name or ""): ledger.available(cluster_name, pool.name or "") for pool in _schedulable_pools(cluster)} placements: list[EnginePlacement] = [] for engine in engines: members = _place_engine(cluster, engine, trial) @@ -808,7 +808,7 @@ def _place_engine( Decrements trial as it places; on failure the caller discards the whole trial, so partial decrements don't leak. """ - pools = _gpu_pools(cluster) + pools = _schedulable_pools(cluster) # Whether each member's requests resolve a claimable device on any pool of # this cluster, ignoring capacity. Used to refuse a placement that would # strand a claim-capable member on a pool where it resolves only synthetic @@ -832,7 +832,7 @@ def _place_engine( def _match_members( - pool: icv1alpha1.GpuPool, engine: _CompiledEngine, claimable: list[bool] + pool: icv1alpha1.SchedulablePool, engine: _CompiledEngine, claimable: list[bool] ) -> list[MemberPlacement] | None: """Match every member of an engine against one pool. diff --git a/functions/compose-model-deployment/tests/test_fn.py b/functions/compose-model-deployment/tests/test_fn.py index 277acd122..b4c38fee1 100644 --- a/functions/compose-model-deployment/tests/test_fn.py +++ b/functions/compose-model-deployment/tests/test_fn.py @@ -173,7 +173,7 @@ def _cluster( } ], "providerConfigRef": {"name": name}, - "gpuPools": [ + "schedulablePools": [ { "name": "default", "nodes": nodes, diff --git a/functions/compose-model-deployment/tests/test_scheduling.py b/functions/compose-model-deployment/tests/test_scheduling.py index 560dee16c..b1816a1a9 100644 --- a/functions/compose-model-deployment/tests/test_scheduling.py +++ b/functions/compose-model-deployment/tests/test_scheduling.py @@ -120,7 +120,7 @@ def _cluster( name: str, gateway_hostname: str | None, ready: bool, - pools: list[icv1alpha1.GpuPool], + pools: list[icv1alpha1.SchedulablePool], taints: list[icv1alpha1.Taint] | None, placement_labels: dict[str, str] | None, ) -> icv1alpha1.InferenceCluster: @@ -152,7 +152,7 @@ def _cluster( # InferenceGateway addresses a cluster by name. gateway=icv1alpha1.Gateway(address="10.0.0.1", hostname=gateway_hostname), providerConfigRef=icv1alpha1.ProviderConfigRef(name=name), - gpuPools=pools, + schedulablePools=pools, ), ) @@ -257,7 +257,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -310,7 +310,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=False, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -342,7 +342,7 @@ def _candidate( gateway_hostname=None, ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -379,7 +379,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -411,7 +411,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -423,7 +423,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -498,7 +498,7 @@ def _candidate( gateway_hostname=None, ready=False, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -573,7 +573,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -648,7 +648,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -660,7 +660,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -755,7 +755,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -830,7 +830,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -903,7 +903,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -915,7 +915,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -988,7 +988,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1000,7 +1000,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1093,7 +1093,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1105,7 +1105,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1198,7 +1198,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1210,7 +1210,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1305,7 +1305,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1441,7 +1441,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1453,7 +1453,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1599,7 +1599,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=6, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1735,7 +1735,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1747,7 +1747,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -1884,7 +1884,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2000,7 +2000,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2012,7 +2012,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2108,7 +2108,7 @@ def _candidate( gateway_hostname="cluster-c.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2120,7 +2120,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2132,7 +2132,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2205,7 +2205,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2259,7 +2259,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2337,7 +2337,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -2417,10 +2417,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), ], @@ -2520,7 +2520,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2552,7 +2552,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2627,7 +2627,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2639,7 +2639,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -2715,7 +2715,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -2770,7 +2770,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -2804,7 +2804,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")], @@ -2841,7 +2841,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=0, memory="141Gi")], @@ -2875,7 +2875,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=0, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -2912,7 +2912,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -2973,7 +2973,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3011,7 +3011,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3050,7 +3050,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], @@ -3087,7 +3087,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")], @@ -3151,7 +3151,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="dev", nodes=2, devices=[ @@ -3159,7 +3159,7 @@ def _candidate( _nic_device(link_type="gpudirect-tcpx"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -3219,7 +3219,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -3256,7 +3256,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3338,7 +3338,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -3346,7 +3346,7 @@ def _candidate( _nic_device(link_type="gpudirect-tcpx"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[ @@ -3429,7 +3429,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -3437,7 +3437,7 @@ def _candidate( _nic_device(link_type="infiniband"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[ @@ -3520,7 +3520,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -3582,7 +3582,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -3666,7 +3666,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -3784,10 +3784,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=4, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=8, memory="141Gi")] ), ], @@ -3847,10 +3847,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="200Gi")] ), ], @@ -3922,10 +3922,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="200Gi")] ), ], @@ -3962,10 +3962,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=8, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="40Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), ], @@ -4002,10 +4002,10 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="small", nodes=8, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="40Gi")] ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ), ], @@ -4017,7 +4017,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="big", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4089,7 +4089,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="a", nodes=2, devices=[ @@ -4097,7 +4097,7 @@ def _candidate( _gpu_device(name="syn", claim="Synthetic", count=1, memory="200Gi"), ], ), - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="b", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="200Gi")] ), ], @@ -4169,7 +4169,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[ @@ -4237,7 +4237,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4270,7 +4270,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -4331,7 +4331,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=1, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], @@ -4416,7 +4416,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4497,7 +4497,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=4, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4588,7 +4588,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4600,7 +4600,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4653,7 +4653,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4728,7 +4728,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4781,7 +4781,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4793,7 +4793,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4868,7 +4868,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -4944,7 +4944,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5000,7 +5000,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5012,7 +5012,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5087,7 +5087,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5102,7 +5102,7 @@ def _candidate( gateway_hostname="cluster-b.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5155,7 +5155,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5208,7 +5208,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5240,7 +5240,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5301,7 +5301,7 @@ def _candidate( gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="default", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")] ) ], @@ -5371,7 +5371,7 @@ def test_schedule_error() -> None: gateway_hostname="cluster-a.clusters.example.com", ready=True, pools=[ - icv1alpha1.GpuPool( + icv1alpha1.SchedulablePool( name="frontier", nodes=2, devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="141Gi")], diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index 136b9aaee..2dbb82f20 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -754,12 +754,22 @@ def engine_resources(container: v1alpha1.Container, *, claim: bool) -> dict: return resources -# Taint GPU node groups carry so non-GPU pods don't land on them. A pod that -# claims a GPU must tolerate it to schedule there. With GPUs bound via DRA (not -# the device plugin's extended resource), nothing injects this toleration for us -# - the ExtendedResourceToleration admission controller only acts on -# nvidia.com/gpu resource requests, which DRA pods don't make. -_GPU_TOLERATION = {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} +# Taints inference node pools carry so unrelated pods don't land on them, one +# per device stack: the cluster compositions taint NVIDIA GPU pools +# nvidia.com/gpu and CPU pools modelplane.ai/cpu, and BYO AMD pools carry +# amd.com/gpu. A pod that claims a device must tolerate its pool's taint to +# schedule there, and with devices bound via DRA (not a device plugin's +# extended resource) nothing injects the toleration for us - the +# ExtendedResourceToleration admission controller only acts on nvidia.com/gpu +# resource requests, which DRA pods don't make. Every engine pod tolerates the +# whole list: tolerating a taint its pool never carries is harmless, and the +# pool nodeSelector is what places the pod, so the accelerator needn't thread +# through placement. +_POOL_TOLERATIONS = [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, +] # Node label identifying the pool a node belongs to. compose-eks-cluster and # compose-gke-cluster stamp it on every node group they provision; the scheduler @@ -774,9 +784,9 @@ def place_pod(pod_spec: dict, replica: v1alpha1.ModelReplica, engine: v1alpha1.E """Constrain a member's serving pod to the placement the scheduler chose. Pins the pod to its member's scheduled node pool, wires it to claim its - GPUs via DRA through the member's claim, and tolerates the GPU node taint. - Every pod of one member shares this - a native Deployment pod, a - LeaderWorkerSet leader or worker pod, or a Grove leader or worker clique + devices via DRA through the member's claim, and tolerates every inference + pool taint. Every pod of one member shares this - a native Deployment pod, + a LeaderWorkerSet leader or worker pod, or a Grove leader or worker clique pod. The pool nodeSelector is what makes the scheduler's pool choice real: the @@ -793,14 +803,14 @@ def place_pod(pod_spec: dict, replica: v1alpha1.ModelReplica, engine: v1alpha1.E claim. A claimless member - one with no deviceRequests, like a coordinator-only leader - gets no claim at all: only the pool pin places its pods, packed onto the gang's nodes by the cluster's scheduler. It still - tolerates the GPU taint, since the pool it rides along on is a GPU pool. + tolerates the pool taints, since the pool it rides along on carries one. """ pod_spec["nodeSelector"] = {_LABEL_POOL: member.nodePoolName} if member.deviceRequests: pod_spec["resourceClaims"] = [ {"name": _POD_CLAIM_NAME, "resourceClaimTemplateName": claim_template_name(replica, engine, member)} ] - pod_spec.setdefault("tolerations", []).append(_GPU_TOLERATION) + pod_spec.setdefault("tolerations", []).extend(_POOL_TOLERATIONS) def resource_claim_template( diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index f57629dd3..4e6ee7b99 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -455,7 +455,9 @@ def _deployment( } ], "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, }, @@ -516,7 +518,9 @@ def _leader_worker_set( } ], "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, }, @@ -535,7 +539,9 @@ def _leader_worker_set( } ], "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, }, @@ -2079,7 +2085,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2129,7 +2145,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2511,7 +2537,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2561,7 +2597,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2730,7 +2776,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2780,7 +2836,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2949,7 +3015,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3001,7 +3077,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3142,7 +3228,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3192,7 +3288,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3349,7 +3455,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3399,7 +3515,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3578,7 +3704,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3660,7 +3796,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3850,7 +3996,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -3932,7 +4088,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -4128,7 +4294,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -4214,7 +4390,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -6542,7 +6728,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -6592,7 +6788,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -6838,7 +7044,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -6888,7 +7104,17 @@ def test_cache_env(case: CacheEnvCase) -> None: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, diff --git a/functions/compose-model-replica/tests/test_fn.py b/functions/compose-model-replica/tests/test_fn.py index 3d64a4f60..1be4b9e39 100644 --- a/functions/compose-model-replica/tests/test_fn.py +++ b/functions/compose-model-replica/tests/test_fn.py @@ -267,7 +267,17 @@ def _deployment(*, ready: fnv1.Ready, name: str, claim_template_name: str, label "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, diff --git a/functions/compose-nebius-cluster/function/fn.py b/functions/compose-nebius-cluster/function/fn.py index 036a8c27b..5ffb82493 100644 --- a/functions/compose-nebius-cluster/function/fn.py +++ b/functions/compose-nebius-cluster/function/fn.py @@ -102,6 +102,12 @@ _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NO_SCHEDULE" +# Label and taint on CPU inference groups. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the group as the GPU taint does on GPU groups, and +# engine pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Management policies that exclude Delete, used for resources installed on the # workload cluster (the RWX StorageClass Object and the CSI driver Helm # Release). These exist only to configure the cluster and are only ever @@ -552,6 +558,8 @@ def compose_node_groups(self) -> None: labels = {_LABEL_POOL: pool.name} if pool.role == "GPU" and pool.gpu: labels[_LABEL_GPU] = pool.gpu.acceleratorType + elif pool.role == "CPU": + labels[_LABEL_CPU] = "true" template = nodegroupv1beta1.Template( resources=nodegroupv1beta1.Resources( @@ -585,6 +593,14 @@ def compose_node_groups(self) -> None: effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + template.taints = [ + nodegroupv1beta1.Taint( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] fabric = _pool_fabric(pool) if fabric: diff --git a/functions/compose-nebius-cluster/tests/test_fn.py b/functions/compose-nebius-cluster/tests/test_fn.py index 5f61cc045..5f43648f8 100644 --- a/functions/compose-nebius-cluster/tests/test_fn.py +++ b/functions/compose-nebius-cluster/tests/test_fn.py @@ -312,6 +312,55 @@ def _nodegroup_gpu( ) +def _nodegroup_cpu(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: + """The composed node group for the cpu-pool pool, with the CPU label and taint and no GPU settings.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "mk8s.nebius.m.upbound.io/v1beta1", + "kind": "NodeGroup", + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "forProvider": { + "name": "test-cluster-cpu-pool", + "parentIdSelector": {"matchControllerRef": True}, + "version": "1.34", + "template": { + "resources": {"platform": "cpu-d3", "preset": "8vcpu-32gb"}, + "bootDisk": {"sizeGibibytes": 100, "type": "NETWORK_SSD"}, + "networkInterfaces": [ + {"subnetIdSelector": {"matchControllerRef": True}}, + ], + "filesystems": [ + { + "attachMode": "READ_WRITE", + "mountTag": "modelplane-cache", + "existingFilesystem": {"idSelector": {"matchControllerRef": True}}, + }, + ], + "cloudInitUserDataSecretRef": { + "name": "test-cluster-cloud-init-fd2f2", + "key": "userData", + }, + "metadata": { + "labels": { + "modelplane.ai/pool": "cpu-pool", + "modelplane.ai/cpu": "true", + }, + }, + "taints": [ + {"key": "modelplane.ai/cpu", "value": "true", "effect": "NO_SCHEDULE"}, + ], + }, + "autoscaling": {"minNodeCount": 1, "maxNodeCount": 4}, + }, + }, + } + ), + ready=ready, + ) + + def _provider_config(*, api_version: str) -> fnv1.Resource: """A Ready ProviderConfig targeting the cluster as the Nebius service account.""" return fnv1.Resource( @@ -431,6 +480,73 @@ def _to_dict(msg: message.Message) -> dict: ), ), ), + Case( + name="FirstPassCPUPool", + reason="A CPU-role node group composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, and no GPU settings.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(credentials_secret=True), + resources={ + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cloud-init": _cloud_init_secret(), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), Case( name="ProviderConfigNotFetched", reason="Before the Nebius provider config is fetched, and with no ProviderConfig observed, a NebiusCluster composes its infrastructure but not the ProviderConfigs, and reports it's waiting.", diff --git a/functions/compose-serving-stack/function/fn.py b/functions/compose-serving-stack/function/fn.py index e58cf0b70..78e9830dc 100644 --- a/functions/compose-serving-stack/function/fn.py +++ b/functions/compose-serving-stack/function/fn.py @@ -327,9 +327,19 @@ def compose(self) -> None: # The XRD requires and enums both fields, so the join raising means the # API and the stacks package disagree on a value, a broken Modelplane - # build, not a cluster condition. Let it crash rather than dress it up - # as a fatal result. - components = stacks.join(self.xr.spec.cloud, self.xr.spec.stack or "Standard") + # build, not a cluster condition - or, for a draDrivers entry, that + # compose-inference-cluster's hand-kept mirror of the supported + # drivers drifted from the stacks package's: it should have gated the + # cluster with UnsupportedDevices before composing this ServingStack. + # Let it crash rather than dress it up as a fatal result. draDrivers + # absent (a ServingStack from before the field) means no filtering, + # so the full stack keeps running until the cluster composition + # stamps the list. + components = stacks.join( + self.xr.spec.cloud, + self.xr.spec.stack or "Standard", + dra_drivers=None if self.xr.spec.draDrivers is None else [d.root for d in self.xr.spec.draDrivers], + ) # Per-pool GPU driver configuration is per-cluster state the cluster # composition projects into spec.gpu, so it can't live in the diff --git a/functions/compose-serving-stack/function/stacks/__init__.py b/functions/compose-serving-stack/function/stacks/__init__.py index 5e803c978..985802e0e 100644 --- a/functions/compose-serving-stack/function/stacks/__init__.py +++ b/functions/compose-serving-stack/function/stacks/__init__.py @@ -39,6 +39,7 @@ "join", "metrics", "stacks", + "supported_drivers", ] # The cloud halves, keyed by the InferenceCluster's source values. EKS, @@ -59,6 +60,23 @@ "Dynamo": dynamo.COMPONENTS, } +# DRA drivers a cloud supports that arrive outside the serving stack, so +# the filter installs nothing for them. The generated modules' constants +# are emitted by generate.py, where the EKS dra.net declaration lives +# beside the aws-efa DROP entry that records why. Existing is the +# open-ended case and is special-cased in join() instead: the user owns +# the cluster, so every driver its list doesn't carry is treated as +# externally installed. +_EXTERNAL_DRIVERS: dict[Cloud, frozenset[str]] = { + "EKS": frozenset(eks.EXTERNAL_DRIVERS), + "AKS": frozenset(aks.EXTERNAL_DRIVERS), + "GKE": frozenset(gke.EXTERNAL_DRIVERS), + "Nebius": frozenset(), + "Vultr": frozenset(), + "Civo": frozenset(), + "Existing": frozenset(), +} + def clouds() -> list[Cloud]: """The clouds a stack can be joined for.""" @@ -70,14 +88,46 @@ def stacks() -> list[Stack]: return list(_STACKS) -def join(cloud: Cloud, stack: Stack) -> list[Component]: +def supported_drivers(cloud: Cloud) -> frozenset[str]: + """The DRA drivers a cloud's serving stack supports. + + Derived rather than declared: the union of the dra_drivers tags + across the cloud's component list, common.py and every stack half, + plus the drivers the cloud declares installed outside the stack + (_EXTERNAL_DRIVERS), so the table can't drift from what the lists + record. compose-inference-cluster mirrors this by hand in its + _SUPPORTED_DRA_DRIVERS to reject unsupported pairings with a + condition before a ServingStack is ever composed - keep the two in + sync. Existing's tags are returned like any cloud's, but a driver + beyond them is not unsupported there: the user owns the cluster, so + join() never rejects on Existing. + """ + halves = [_CLOUDS[cloud], common.COMPONENTS, *_STACKS.values()] + tagged = {driver for half in halves for c in half for driver in c.dra_drivers} + return frozenset(tagged) | _EXTERNAL_DRIVERS[cloud] + + +def join(cloud: Cloud, stack: Stack, dra_drivers: list[str] | None = None) -> list[Component]: """Join the component lists for a cloud and stack. Fails closed, at import or test time rather than on a cluster: on an - unknown cloud or stack, on a key two lists both produce, and on a + unknown cloud or stack, on a key two lists both produce, on a depends_on edge naming a component the join didn't produce - which catches a generator allowlist that dropped something another - component needs. + component needs - and on a dependency that is neither untagged nor + tagged with every driver its dependent carries, which filtering + could then remove from under it. + + dra_drivers filters the tagged components: a tagged component + survives only when one of its drivers is listed, an untagged one + always does. None (the field unset on the XR) disables filtering, so + a ServingStack from before the field existed keeps today's full + stack rather than tearing its device stacks down. The integrity + checks run on the unfiltered join, so a broken list fails every join + for its cloud, not just the driver combination that trips it. A + listed driver the cloud neither tags components with nor declares + external raises - except on Existing, where every such driver is + treated as externally installed and installs nothing. """ if cloud not in _CLOUDS: raise ValueError(f"unknown cloud {cloud!r}; known: {', '.join(_CLOUDS)}") @@ -99,10 +149,32 @@ def join(cloud: Cloud, stack: Stack) -> list[Component]: if duplicates: raise ValueError(f"{cloud}/{stack}: duplicate composed-resource keys {duplicates}") - known = set(keys) + by_key = {c.key: c for c in joined} for c in joined: for dep in c.depends_on: - if dep not in known: + if dep not in by_key: raise ValueError(f"{cloud}/{stack}: {c.key} depends on {dep!r}, which the join did not produce") - - return joined + # A dependency must survive every filter that keeps its + # dependent: untagged, or tagged with every driver the + # dependent carries - which an untagged dependent, kept by + # every filter, can never satisfy. + dep_drivers = by_key[dep].dra_drivers + if dep_drivers and not (c.dra_drivers and set(c.dra_drivers) <= set(dep_drivers)): + raise ValueError( + f"{cloud}/{stack}: {c.key} (drivers {c.dra_drivers or 'untagged'}) depends on " + f"{dep!r} (drivers {dep_drivers}), which filtering could remove from under it" + ) + + if dra_drivers is None: + return joined + + if cloud != "Existing": + unknown = set(dra_drivers) - supported_drivers(cloud) + if unknown: + raise ValueError( + f"{cloud}/{stack}: no device stack serves {', '.join(sorted(unknown))}; " + f"the {cloud} serving stack supports {', '.join(sorted(supported_drivers(cloud)))}" + ) + + wanted = set(dra_drivers) + return [c for c in joined if not c.dra_drivers or wanted & set(c.dra_drivers)] diff --git a/functions/compose-serving-stack/function/stacks/clouds/civo.py b/functions/compose-serving-stack/function/stacks/clouds/civo.py index 654f10816..e31a224ad 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/civo.py +++ b/functions/compose-serving-stack/function/stacks/clouds/civo.py @@ -220,19 +220,21 @@ def transformed(c: Component) -> Component: ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the GPU and CPU pool + # taints, # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), Chart( @@ -245,6 +247,10 @@ def transformed(c: Component) -> Component: # gpu-operator below depends on this chart, so its Ready must # mean healthy, not just deployed. wait=True, + # NFD exists here to label GPU nodes for the DRA driver, so it + # carries the GPU stack's driver and stays off clusters whose + # classes claim no GPUs. + dra_drivers=["gpu.nvidia.com"], # The worker must run on the very nodes it is supposed to label: # the DRA driver's kubelet plugin schedules only onto nodes # carrying an NFD GPU label (feature.node.kubernetes.io/pci-10de @@ -287,6 +293,7 @@ def transformed(c: Component) -> Component: # modelplane_energy_joules_total is empty on Civo - the # collector's modelplane-gpu job would find no pod to scrape. depends_on=["node-feature-discovery", "cert-manager"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -363,6 +370,7 @@ def transformed(c: Component) -> Component: repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "nvidiaDriverRoot": "/run/nvidia/driver", diff --git a/functions/compose-serving-stack/function/stacks/clouds/existing.py b/functions/compose-serving-stack/function/stacks/clouds/existing.py index 6fc8b5830..cf064295f 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/existing.py +++ b/functions/compose-serving-stack/function/stacks/clouds/existing.py @@ -20,6 +20,15 @@ component also appears on a generated cloud, this file states the same pin, so one review moves both halves when a version changes - regenerate, then mirror the shared pins here. + +Existing is also the open-ended cloud for devices: the user owns the +cluster and may have installed any DRA driver on it, so the join treats +a claimed driver this list doesn't tag as externally installed - it +installs nothing and rejects nothing. The list still pins device +stacks like any cloud's, each tagged with its driver so it installs +only when a class claims it: NVIDIA's today, and AMD's - Existing is +the one cloud that installs it, for BYO nodes whose images already +carry the amdgpu kernel driver. """ from function.stacks.components import Chart, Component @@ -90,19 +99,22 @@ ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the inference pool + # taints - a BYO pool may carry any of the three - # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), Chart( @@ -112,16 +124,24 @@ chart="node-feature-discovery", repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", + # amd-gpu-operator below depends on this chart, so its Ready must + # mean healthy, not just deployed. + wait=True, + # NFD exists here to label GPU nodes for the DRA drivers, so it + # carries both GPU stacks' drivers - NVIDIA's kubelet plugin and + # AMD's operator both find their nodes by NFD labels - and stays + # off clusters whose classes claim no GPUs. + dra_drivers=["gpu.nvidia.com", "gpu.amd.com"], # The worker must run on the very nodes it is supposed to label: # the DRA driver's kubelet plugin schedules only onto nodes # carrying an NFD GPU label (feature.node.kubernetes.io/pci-10de # and friends). The cluster compositions taint GPU nodes with - # nvidia.com/gpu, and the NFD chart's worker tolerates nothing - # by default. Without this toleration the chain breaks silently: - # no worker on the GPU node, no pci-10de label, no kubelet - # plugin, no ResourceSlices, and every GPU ResourceClaim stays - # unallocatable with all components looking healthy. Exists - # matches every taint value. + # nvidia.com/gpu or amd.com/gpu, and the NFD chart's worker + # tolerates nothing by default. Without these tolerations the + # chain breaks silently: no worker on the GPU node, no pci-10de + # label, no kubelet plugin, no ResourceSlices, and every GPU + # ResourceClaim stays unallocatable with all components looking + # healthy. Exists matches every taint value. values={ "worker": { "tolerations": [ @@ -130,6 +150,11 @@ "operator": "Exists", "effect": "NoSchedule", }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -147,9 +172,66 @@ chart="dra-driver-nvidia-gpu", repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "resources": {"computeDomains": {"enabled": False}}, }, ), + # The AMD GPU operator, in DRA mode: the chart's default DeviceConfig + # (crds.defaultCR) enables the DRA driver and disables the device + # plugin - the two are mutually exclusive allocators - so the + # gpu.amd.com DeviceClass the chart registers is the sole path to the + # GPUs. Pin validated on Vultr bare metal (8x MI325X); Existing is + # the one cloud that installs it today. Driver management (KMM) + # stays off: a BYO node's image must already carry the amdgpu kernel + # driver, and the chart's in-cluster driver build needs a push + # registry configured (driver.image defaults to a docker.io + # placeholder), without which KMM loops on a driver image it can + # never resolve and the DRA driver - gated on the KMM-ready node + # label - never schedules. The node labeller stays on for scheduling + # labels; NFD is the standalone instance above, so the chart's + # bundled one is off and two instances never race over the same + # label namespace. Every per-node component tolerates the + # amd.com/gpu taint AMD GPU pools carry - the chart defaults + # tolerate nothing, which strands the daemonsets off the very nodes + # they serve. + Chart( + key="amd-gpu-operator", + release="mp-gpu-operator-charts", + namespace="kube-amd-gpu", + chart="gpu-operator-charts", + repository="https://rocm.github.io/gpu-operator", + version="v1.5.1", + depends_on=["cert-manager", "node-feature-discovery"], + dra_drivers=["gpu.amd.com"], + values={ + "node-feature-discovery": {"enabled": False}, + # Argo-based node remediation pulls a workflow controller the + # serving stack doesn't need. + "remediation": {"enabled": False, "installCRDs": False}, + "deviceConfig": { + "spec": { + "driver": {"enable": False}, + "devicePlugin": { + "enableDevicePlugin": False, + "nodeLabellerTolerations": [ + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + }, + "metricsExporter": { + "tolerations": [ + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + }, + "draDriver": { + "enable": True, + "tolerations": [ + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + }, + }, + }, + }, + ), ] diff --git a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py index 7b2b67fa8..cb1d07e33 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py +++ b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/aks.py @@ -24,6 +24,11 @@ from function.stacks.components import Chart, Component, Manifests +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table (see generate.py's EXTERNAL_DRIVERS). +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", @@ -61,6 +66,7 @@ repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", wait=True, + dra_drivers=["gpu.nvidia.com"], values={ "gc": {"enable": True, "tolerations": []}, "master": {"enable": True, "tolerations": []}, @@ -88,6 +94,7 @@ chart="nodewright", repository="oci://ghcr.io/nvidia/nodewright/charts", version="v0.17.1", + dra_drivers=["gpu.nvidia.com"], values={ "controllerManager": { "manager": { @@ -190,7 +197,10 @@ }, "prometheus-node-exporter": { "fullnameOverride": "prometheus-node-exporter", - "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "tolerations": [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], }, "prometheusOperator": {"admissionWebhooks": {"patch": {"tolerations": []}}, "tolerations": []}, "thanosRuler": {"thanosRulerSpec": {"tolerations": []}}, @@ -205,6 +215,7 @@ version="v26.3.3", wait=True, depends_on=["node-feature-discovery", "cert-manager", "kube-prometheus-stack"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "daemonsets": { @@ -325,6 +336,7 @@ Manifests( key="gpu-operator-manifests", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], manifests=[ { "apiVersion": "apps/v1", @@ -452,6 +464,7 @@ repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "controller": { "affinity": {}, @@ -481,6 +494,7 @@ repository="oci://ghcr.io/nvidia", version="v1.20.0", depends_on=["cert-manager", "gpu-operator", "prometheus-operator-crds"], + dra_drivers=["gpu.nvidia.com"], values={ "fullnameOverride": "nvsentinel", "global": { diff --git a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py index a16047f37..34195901d 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py +++ b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/eks.py @@ -24,6 +24,11 @@ from function.stacks.components import Chart, Component +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table (see generate.py's EXTERNAL_DRIVERS). +EXTERNAL_DRIVERS: list[str] = ["dra.net"] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", @@ -61,6 +66,7 @@ repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", wait=True, + dra_drivers=["gpu.nvidia.com"], values={ "gc": {"enable": True, "tolerations": []}, "master": {"enable": True, "tolerations": []}, @@ -84,6 +90,7 @@ chart="nodewright", repository="oci://ghcr.io/nvidia/nodewright/charts", version="v0.17.1", + dra_drivers=["gpu.nvidia.com"], values={ "controllerManager": { "manager": { @@ -186,7 +193,10 @@ }, "prometheus-node-exporter": { "fullnameOverride": "prometheus-node-exporter", - "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "tolerations": [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], }, "prometheusOperator": {"admissionWebhooks": {"patch": {"tolerations": []}}, "tolerations": []}, "thanosRuler": {"thanosRulerSpec": {"tolerations": []}}, @@ -201,6 +211,7 @@ version="v26.3.3", wait=True, depends_on=["node-feature-discovery", "cert-manager", "kube-prometheus-stack"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "daemonsets": {"tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}]}, @@ -341,6 +352,7 @@ repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "controller": { "affinity": {}, @@ -368,6 +380,7 @@ repository="oci://ghcr.io/nvidia", version="v1.20.0", depends_on=["cert-manager", "gpu-operator", "prometheus-operator-crds"], + dra_drivers=["gpu.nvidia.com"], values={ "fullnameOverride": "nvsentinel", "global": { diff --git a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py index 431c5a4ef..b3f51e17b 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py +++ b/functions/compose-serving-stack/function/stacks/clouds/generated/aicr/gke.py @@ -24,6 +24,11 @@ from function.stacks.components import Chart, Component, Manifests +# DRA drivers this cloud supports that arrive outside the serving +# stack; the stacks package folds them into its derived +# supported-drivers table (see generate.py's EXTERNAL_DRIVERS). +EXTERNAL_DRIVERS: list[str] = [] + COMPONENTS: list[Component] = [ Chart( key="cert-manager", @@ -61,6 +66,7 @@ repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", wait=True, + dra_drivers=["gpu.nvidia.com"], values={ "gc": {"enable": True, "tolerations": []}, "master": {"enable": True, "tolerations": []}, @@ -84,6 +90,7 @@ chart="nodewright", repository="oci://ghcr.io/nvidia/nodewright/charts", version="v0.17.1", + dra_drivers=["gpu.nvidia.com"], values={ "controllerManager": { "manager": { @@ -187,7 +194,10 @@ }, "prometheus-node-exporter": { "fullnameOverride": "prometheus-node-exporter", - "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "tolerations": [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], }, "prometheusOperator": {"admissionWebhooks": {"patch": {"tolerations": []}}, "tolerations": []}, "thanosRuler": {"thanosRulerSpec": {"tolerations": []}}, @@ -195,6 +205,7 @@ ), Manifests( key="gpu-operator-pre-manifests", + dra_drivers=["gpu.nvidia.com"], manifests=[ {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": "gpu-operator"}}, { @@ -225,6 +236,7 @@ version="v26.3.3", wait=True, depends_on=["node-feature-discovery", "cert-manager", "kube-prometheus-stack", "gpu-operator-pre-manifests"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -371,6 +383,7 @@ repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", depends_on=["gpu-operator"], + dra_drivers=["gpu.nvidia.com"], values={ "controller": { "affinity": {}, @@ -398,6 +411,7 @@ repository="oci://ghcr.io/nvidia", version="v1.20.0", depends_on=["cert-manager", "gpu-operator", "prometheus-operator-crds"], + dra_drivers=["gpu.nvidia.com"], values={ "fullnameOverride": "nvsentinel", "global": { diff --git a/functions/compose-serving-stack/function/stacks/clouds/nebius.py b/functions/compose-serving-stack/function/stacks/clouds/nebius.py index aa1f72225..054780e26 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/nebius.py +++ b/functions/compose-serving-stack/function/stacks/clouds/nebius.py @@ -89,19 +89,21 @@ ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the GPU and CPU pool + # taints, # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), Chart( @@ -111,6 +113,10 @@ chart="node-feature-discovery", repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", version="0.19.0", + # NFD exists here to label GPU nodes for the DRA driver, so it + # carries the GPU stack's driver and stays off clusters whose + # classes claim no GPUs. + dra_drivers=["gpu.nvidia.com"], # The worker must run on the very nodes it is supposed to label: # the DRA driver's kubelet plugin schedules only onto nodes # carrying an NFD GPU label (feature.node.kubernetes.io/pci-10de @@ -146,6 +152,7 @@ chart="dra-driver-nvidia-gpu", repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "resources": {"computeDomains": {"enabled": False}}, diff --git a/functions/compose-serving-stack/function/stacks/clouds/vultr.py b/functions/compose-serving-stack/function/stacks/clouds/vultr.py index 4fad84294..479fd027d 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/vultr.py +++ b/functions/compose-serving-stack/function/stacks/clouds/vultr.py @@ -99,19 +99,21 @@ ], }, }, - # Disable components we don't need for observability. - "grafana": {"enabled": False}, - "alertmanager": {"enabled": False}, # The node-exporter chart's default toleration is keyless - it - # tolerates every taint, landing the pod on tainted GPU nodes by - # accident rather than by intent. Scope it to the GPU taint, + # tolerates every taint, landing the pod on tainted nodes by + # accident rather than by intent. Scope it to the GPU and CPU pool + # taints, # mirroring the generated clouds (TOLERATIONS in generate.py); as # a DaemonSet it still reaches every untainted node. "prometheus-node-exporter": { "tolerations": [ {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, ], }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, }, ), # Publishes each GPU node's devices as DRA ResourceSlices and @@ -127,6 +129,7 @@ chart="dra-driver-nvidia-gpu", repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "resources": {"computeDomains": {"enabled": False}}, diff --git a/functions/compose-serving-stack/function/stacks/common.py b/functions/compose-serving-stack/function/stacks/common.py index 10c5ec3ee..7296b25df 100644 --- a/functions/compose-serving-stack/function/stacks/common.py +++ b/functions/compose-serving-stack/function/stacks/common.py @@ -18,7 +18,9 @@ Dynamo stacks don't differ: the gateway path (Envoy Gateway, Envoy AI Gateway and its CRDs, the Gateway API Inference Extension CRDs, the gateway namespace and EnvoyProxy), the gateway's PKI (a self-signed -Issuer and trust-manager) and the DRA critical-pods quota. +Issuer and trust-manager), the DRA critical-pods quotas, and the device +stacks every cloud pins identically - today dra-driver-cpu, whose nodes +look the same whatever provisioned them. The GatewayClass and Gateway are deliberately absent: they read spec.gateway (className, hostname), which is per-cluster configuration, @@ -26,7 +28,7 @@ Versions here are Modelplane's pins: this file carries what no generator resolves - the contract surface a ModelReplica composes -against, whose versions have nothing to do with the hardware (see +against, whose versions have nothing to do with the GPU hardware (see design/serving-stack-generation.md, "Component sources"). """ @@ -66,7 +68,11 @@ # Must match the namespace every cloud half installs the NVIDIA DRA # driver into - generated and hand-written alike - so this quota lands # where the kubelet plugin runs. -_DRA_DRIVER_NAMESPACE = "nvidia-dra-driver" +_NVIDIA_DRA_DRIVER_NAMESPACE = "nvidia-dra-driver" + +# The namespace the dra-driver-cpu component below installs into; its +# critical-pods quota must land in the same place. +_CPU_DRA_DRIVER_NAMESPACE = "dra-driver-cpu" _CRDS_DIR = pathlib.Path(__file__).parent / "crds" @@ -274,17 +280,79 @@ def _crds(filename: str) -> list[dict[str, Any]]: # priority. GKE only admits such pods in a namespace whose # ResourceQuota permits those priority classes; without it the # daemonset gets FailedCreate and never publishes ResourceSlices. - # Laid down everywhere: it only grants headroom, so it's harmless on - # clusters that don't restrict them. + # Laid down wherever the NVIDIA stack installs: it only grants + # headroom, so it's harmless on clusters that don't restrict them, + # but the tag keeps it off clusters where nothing creates the + # nvidia-dra-driver namespace, where the Object could never apply. Manifests( key="dra-driver-critical-pods-quota", + dra_drivers=["gpu.nvidia.com"], + manifests=[ + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "metadata": { + "name": "allow-critical-pods", + "namespace": _NVIDIA_DRA_DRIVER_NAMESPACE, + }, + "spec": { + "hard": {"pods": "1000"}, + "scopeSelector": { + "matchExpressions": [ + { + "operator": "In", + "scopeName": "PriorityClass", + "values": [ + "system-node-critical", + "system-cluster-critical", + ], + }, + ], + }, + }, + }, + ], + ), + # Publishes each CPU node's cores as DRA devices and registers the + # dra.cpu DeviceClass CPU classes claim through: one device per NUMA + # node (the chart's default grouped mode), with the node's + # allocatable CPUs as consumable capacity. A claimed device pins the + # engine container to a cpuset. Pinned here because a CPU node needs + # the same driver whatever provisioned it, so every cloud - Existing + # included - installs the one pin. The nodeSelector keeps it to CPU + # pools, which every cluster composition labels modelplane.ai/cpu: + # the chart's default tolerations accept every NoSchedule taint, so + # the selector is what keeps it off GPU and system pools, not their + # taints. reservedCPUs holds cores 0-1 back for the OS and kubelet, + # so a class's capacity.cpu is the node's vCPUs less two - a fixed, + # documented value rather than a per-pool knob someone must keep in + # step with the class by hand. + Chart( + key="dra-driver-cpu", + release="mp-dra-driver-cpu", + namespace=_CPU_DRA_DRIVER_NAMESPACE, + chart="dra-driver-cpu", + repository="oci://registry.k8s.io/dra-driver-cpu/charts", + version="0.3.0", + dra_drivers=["dra.cpu"], + values={ + "nodeSelector": {"modelplane.ai/cpu": "true"}, + "driverConfig": {"reservedCPUs": "0-1"}, + }, + ), + # dra-driver-cpu's daemonset runs at system-node-critical priority + # like the NVIDIA kubelet plugin above, so its namespace needs the + # same quota on GKE. + Manifests( + key="dra-driver-cpu-critical-pods-quota", + dra_drivers=["dra.cpu"], manifests=[ { "apiVersion": "v1", "kind": "ResourceQuota", "metadata": { "name": "allow-critical-pods", - "namespace": _DRA_DRIVER_NAMESPACE, + "namespace": _CPU_DRA_DRIVER_NAMESPACE, }, "spec": { "hard": {"pods": "1000"}, diff --git a/functions/compose-serving-stack/function/stacks/components.py b/functions/compose-serving-stack/function/stacks/components.py index cc6267681..09721c4b4 100644 --- a/functions/compose-serving-stack/function/stacks/components.py +++ b/functions/compose-serving-stack/function/stacks/components.py @@ -56,6 +56,15 @@ class Chart: depends on, so the install gate orders on health rather than deploy - the generator derives it from the dependency edges, and the hand-written files state it where a cross-half edge lands on them. + + `dra_drivers` names the DRA drivers whose device stacks this + component belongs to (gpu.nvidia.com, dra.cpu, ...). An untagged + component is needed regardless of devices and always installs; a + tagged one installs only when one of its drivers is in ServingStack + spec.draDrivers. A component can serve several stacks (NFD labels + nodes for every GPU driver), and a dependency must be untagged or + carry every driver its dependent carries, so filtering can never + remove it from under a component it keeps. """ key: str @@ -67,6 +76,7 @@ class Chart: wait: bool = False depends_on: list[str] = field(default_factory=list) values: dict[str, Any] | None = None + dra_drivers: list[str] = field(default_factory=list) @dataclass @@ -82,12 +92,15 @@ class Manifests: applied to every doc in the entry (see fn.py's _k8s_object): use it when readiness must reflect a controller-populated status field, and keep an entry to one doc when only that doc has one. + + `dra_drivers` behaves as on Chart. """ key: str manifests: list[dict[str, Any]] depends_on: list[str] = field(default_factory=list) ready: str | None = None + dra_drivers: list[str] = field(default_factory=list) # A plain assignment rather than a `type` statement: the packages diff --git a/functions/compose-serving-stack/generate.py b/functions/compose-serving-stack/generate.py index 044fd0842..0a123f07f 100644 --- a/functions/compose-serving-stack/generate.py +++ b/functions/compose-serving-stack/generate.py @@ -423,6 +423,14 @@ def tolerations_key(key: str) -> bool: # AKS; GKE's own taint carries present). GPU: list[Values] = [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}] +# Tolerate every schedulable pool's taint - GPU and CPU alike - for +# node-reaching DaemonSets that aren't part of any one device stack and +# must observe every node Modelplane schedules models onto. +NODE: list[Values] = [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, +] + # Recipe component name -> dotted values path -> the tolerations that # replace a wildcard found there. Like ALLOW/DROP this fails closed: a # wildcard at an unclassified path fails the run, else an aicr bump @@ -444,7 +452,7 @@ def tolerations_key(key: str) -> bool: "gc.tolerations": SYSTEM, }, "kube-prometheus-stack": { - "prometheus-node-exporter.tolerations": GPU, + "prometheus-node-exporter.tolerations": NODE, "alertmanager.alertmanagerSpec.tolerations": SYSTEM, "grafana.tolerations": SYSTEM, "kube-state-metrics.tolerations": SYSTEM, @@ -490,22 +498,28 @@ def tolerations_key(key: str) -> bool: ("gpu-operator", "DaemonSet", "nvidia-toolkit-hardening"): GPU, } -# Recipe component name -> the key we emit it as. The key and the -# mp-prefixed release name are Modelplane's, not AICR's, so a -# component's identity survives upstream renames (see the design's -# "Ordering and identity"). +# Recipe component name -> (the key we emit it as, the DRA drivers +# whose device stacks it belongs to). The key and the mp-prefixed +# release name are Modelplane's, not AICR's, so a component's identity +# survives upstream renames (see the design's "Ordering and identity"). +# The driver list fails closed like the allowlist itself: classifying a +# new component includes deciding which stacks it serves, the review a +# bump already requires - the tag just records it. An empty list means +# the component is needed regardless of devices and always installs; a +# tagged component installs only when ServingStack spec.draDrivers +# names one of its drivers (see design/multi-accelerator.md). ALLOW = { - "nfd": "node-feature-discovery", - "cert-manager": "cert-manager", - "kube-prometheus-stack": "kube-prometheus-stack", - "prometheus-operator-crds": "prometheus-operator-crds", - "prometheus-adapter": "prometheus-adapter", - "k8s-ephemeral-storage-metrics": "k8s-ephemeral-storage-metrics", - "gpu-operator": "gpu-operator", - "nvidia-dra-driver-gpu": "nvidia-dra-driver-gpu", - "nvsentinel": "nvsentinel", - "nodewright-operator": "nodewright-operator", - "nodewright-customizations": "nodewright-customizations", + "nfd": ("node-feature-discovery", ["gpu.nvidia.com"]), + "cert-manager": ("cert-manager", []), + "kube-prometheus-stack": ("kube-prometheus-stack", []), + "prometheus-operator-crds": ("prometheus-operator-crds", []), + "prometheus-adapter": ("prometheus-adapter", []), + "k8s-ephemeral-storage-metrics": ("k8s-ephemeral-storage-metrics", []), + "gpu-operator": ("gpu-operator", ["gpu.nvidia.com"]), + "nvidia-dra-driver-gpu": ("nvidia-dra-driver-gpu", ["gpu.nvidia.com"]), + "nvsentinel": ("nvsentinel", ["gpu.nvidia.com"]), + "nodewright-operator": ("nodewright-operator", ["gpu.nvidia.com"]), + "nodewright-customizations": ("nodewright-customizations", ["gpu.nvidia.com"]), } # In the recipe, deliberately not in the stack. Each entry needs a @@ -520,6 +534,16 @@ def tolerations_key(key: str) -> bool: "kai-scheduler": "contract surface, not hardware surface; Modelplane's pin beside its Queues, in stacks/dynamo.py", } +# DRA drivers a cloud supports that arrive outside the serving stack, +# emitted into the generated module so the stacks package's derived +# supported-drivers table can't drift from the generated file. EKS's +# dra.net is the aws-efa DROP entry above as the filter sees it: +# compose-eks-cluster installs the EFA DRA driver when a pool asks for +# the fabric, and the kimi-k2 and qwen3-coder recipes claim it today. +EXTERNAL_DRIVERS = { + "eks": ["dra.net"], +} + # Long recipe-carried config blobs keep their content, not their # cosmetics: dcgm-exporter's metrics CSV ships with blank spacer lines # the exporter ignores. Trimming them keeps the generated file @@ -910,7 +934,7 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis continue if name not in ALLOW: sys.exit(f"unknown component {name!r}: classify it in ALLOW or DROP") - key = ALLOW[name] + key, dra_drivers = ALLOW[name] values = deep_merge(bundle_values(bundle_dir, name), ref.get("overrides")) for path in TRIM.get(name, []): @@ -944,7 +968,7 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis depends = [] for dep in ref.get("dependencyRefs", []): if dep in ALLOW: - depends.append(ALLOW[dep]) + depends.append(ALLOW[dep][0]) elif dep in DROP: findings.append( f"dropped dep: {name} -> {dep} (dropped; the join with " @@ -980,13 +1004,16 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis findings.append( f"pre-manifests: {name}: {len(manifests)} objects (namespace + {', '.join(f.name for f in pre)})" ) - components.append( - { - "type": "Manifests", - "key": f"{key}-pre-manifests", - "manifests": manifests, - } - ) + pre_entry = { + "type": "Manifests", + "key": f"{key}-pre-manifests", + "manifests": manifests, + } + # A dependency must carry every driver its dependent does, + # or the filter could remove it from under the chart. + if dra_drivers: + pre_entry["dra_drivers"] = dra_drivers + components.append(pre_entry) depends.append(f"{key}-pre-manifests") entry = { @@ -1002,6 +1029,8 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis } if depends: entry["depends_on"] = depends + if dra_drivers: + entry["dra_drivers"] = dra_drivers if values: entry["values"] = values components.append(entry) @@ -1023,14 +1052,17 @@ def transform(cloud: str, recipe: Values, bundle_dir: pathlib.Path) -> tuple[lis for doc in manifests: findings += scope_manifest_tolerations(name, doc) findings.append(f"manifests: {name}: {len(manifests)} objects ({', '.join(f.name for f in rendered)})") - components.append( - { - "type": "Manifests", - "key": f"{key}-manifests", - "depends_on": [key], - "manifests": manifests, - } - ) + post_entry = { + "type": "Manifests", + "key": f"{key}-manifests", + "depends_on": [key], + "manifests": manifests, + } + # Depending on a tagged chart requires carrying its tags, so + # the filter keeps or drops the pair together. + if dra_drivers: + post_entry["dra_drivers"] = dra_drivers + components.append(post_entry) # The bundle is consumed fail-closed, like the recipe: every # component directory must belong to a classified component and use @@ -1148,6 +1180,7 @@ def emit( "version", "wait", "depends_on", + "dra_drivers", "values", "manifests", ] @@ -1169,6 +1202,11 @@ def emit( "", f"from function.stacks.components import {', '.join(names)}", "", + "# DRA drivers this cloud supports that arrive outside the serving", + "# stack; the stacks package folds them into its derived", + "# supported-drivers table (see generate.py's EXTERNAL_DRIVERS).", + f"EXTERNAL_DRIVERS: list[str] = {pyfmt(EXTERNAL_DRIVERS.get(cloud, []))}", + "", "COMPONENTS: list[Component] = [", ] for entry in components: diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index 994c76bde..7ad56fb7d 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -95,13 +95,21 @@ def _serving_stack( secrets: list[v1alpha1.Secret], gateway: v1alpha1.Gateway, gpu: v1alpha1.Gpu | None, + dra_drivers: list[str] | None, ) -> fnv1.Resource: - """The observed ServingStack, test-backend in namespace test-ns.""" + """The observed ServingStack, test-backend in namespace test-ns. + + dra_drivers populates spec.draDrivers; None leaves the field absent, + which disables filtering (a ServingStack from before the field). + """ + spec = v1alpha1.Spec(cloud=cloud, stack=stack, secrets=secrets, gateway=gateway, gpu=gpu) + if dra_drivers is not None: + spec.draDrivers = [v1alpha1.DraDriver(d) for d in dra_drivers] return fnv1.Resource( resource=resource.dict_to_struct( v1alpha1.ServingStack( metadata=metav1.ObjectMeta(name="test-backend", namespace="test-ns"), - spec=v1alpha1.Spec(cloud=cloud, stack=stack, secrets=secrets, gateway=gateway, gpu=gpu), + spec=spec, ).model_dump(exclude_none=True, mode="json", by_alias=True) ) ) @@ -292,8 +300,13 @@ def _cert_manager(*, ready: fnv1.Ready) -> fnv1.Resource: """ -def _kube_prometheus_stack(*, ready: fnv1.Ready) -> fnv1.Resource: - """The composed kube-prometheus-stack Release.""" +def _kube_prometheus_stack(*, ready: fnv1.Ready, node_exporter_tolerations: list[dict]) -> fnv1.Resource: + """The composed kube-prometheus-stack Release. + + node_exporter_tolerations is the inference pool taints the cloud's pin + lets the node exporter tolerate: every taint a BYO pool may carry on + Existing, the NVIDIA and CPU taints elsewhere. + """ return fnv1.Resource( resource=resource.dict_to_struct( { @@ -348,13 +361,9 @@ def _kube_prometheus_stack(*, ready: fnv1.Ready) -> fnv1.Resource: ], } }, + "prometheus-node-exporter": {"tolerations": node_exporter_tolerations}, "grafana": {"enabled": False}, "alertmanager": {"enabled": False}, - "prometheus-node-exporter": { - "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, - ], - }, }, }, }, @@ -364,11 +373,12 @@ def _kube_prometheus_stack(*, ready: fnv1.Ready) -> fnv1.Resource: ) -def _node_feature_discovery(*, ready: fnv1.Ready, wait: bool) -> fnv1.Resource: +def _node_feature_discovery(*, ready: fnv1.Ready, wait: bool, worker_tolerations: list[dict]) -> fnv1.Resource: """The composed node-feature-discovery Release the hand-written clouds pin, waiting for health if wait is set. - Civo's waits and Existing's doesn't. GKE's generated half pins its own, - _gke_node_feature_discovery. + worker_tolerations is the GPU pool taints the cloud's pin lets the + worker tolerate: NVIDIA's and AMD's on Existing, NVIDIA's elsewhere. + GKE's generated half pins its own, _gke_node_feature_discovery. """ for_provider: dict = { "chart": { @@ -377,9 +387,7 @@ def _node_feature_discovery(*, ready: fnv1.Ready, wait: bool) -> fnv1.Resource: "version": "0.19.0", }, "namespace": "node-feature-discovery", - "values": { - "worker": {"tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}]} - }, + "values": {"worker": {"tolerations": worker_tolerations}}, } if wait: for_provider |= {"wait": True, "waitTimeout": "10m"} @@ -1837,6 +1845,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -2002,6 +2011,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -2045,8 +2055,22 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -2420,6 +2444,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -2511,8 +2536,22 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "cert-manager": _cert_manager(ready=fnv1.READY_TRUE), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_TRUE), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_TRUE, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_TRUE, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_TRUE, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_TRUE), "envoy-gateway": fnv1.Resource( resource=resource.dict_to_struct( @@ -3155,6 +3194,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Nebius", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret( @@ -3276,6 +3316,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway( # A full Service FQDN, so the CA certificate's commonName overflows @@ -3310,8 +3351,22 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -3439,6 +3494,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway( hostname="gateway-test-backend-12345.modelplane-system.svc.cluster.local", @@ -3481,8 +3537,22 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -3605,6 +3675,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway(hostname="gw.clusters.example.com"), gpu=None, @@ -3623,8 +3694,22 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=False), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "nvidia-dra-driver-gpu": _nvidia_dra_driver_gpu(ready=fnv1.READY_UNSPECIFIED), "ai-gateway-crds": _ai_gateway_crds(ready=fnv1.READY_UNSPECIFIED), "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _gaie_crd( @@ -3741,6 +3826,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Civo", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway(hostname="test-backend.gateways.example.com"), gpu=v1alpha1.Gpu(pools=[v1alpha1.Pool(name="h100-pool", disableNvLink=True)]), @@ -3764,8 +3850,20 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_TRUE), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_TRUE, wait=True), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_TRUE, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "gpu-operator": fnv1.Resource( resource=resource.dict_to_struct( { @@ -4161,6 +4259,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="Civo", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig")], gateway=v1alpha1.Gateway(hostname="test-backend.gateways.example.com"), gpu=v1alpha1.Gpu(pools=[v1alpha1.Pool(name="l40s-pool", disableNvLink=False)]), @@ -4180,8 +4279,20 @@ def _to_dict(msg: message.Message) -> dict: "provider-config-kubernetes": _kubernetes_provider_config(identity=None, ready=fnv1.READY_TRUE), "provider-config-helm": _helm_provider_config(identity=None, ready=fnv1.READY_TRUE), "cert-manager": _cert_manager(ready=fnv1.READY_UNSPECIFIED), - "kube-prometheus-stack": _kube_prometheus_stack(ready=fnv1.READY_UNSPECIFIED), - "node-feature-discovery": _node_feature_discovery(ready=fnv1.READY_UNSPECIFIED, wait=True), + "kube-prometheus-stack": _kube_prometheus_stack( + ready=fnv1.READY_UNSPECIFIED, + node_exporter_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), + "node-feature-discovery": _node_feature_discovery( + ready=fnv1.READY_UNSPECIFIED, + wait=True, + worker_tolerations=[ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], + ), "gpu-operator": fnv1.Resource( resource=resource.dict_to_struct( { @@ -4430,6 +4541,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -4733,6 +4845,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -5100,6 +5213,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -5409,6 +5523,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -5721,6 +5836,7 @@ def _to_dict(msg: message.Message) -> dict: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6071,6 +6187,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="EKS", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6212,6 +6329,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="EKS", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6381,6 +6499,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="AKS", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6527,6 +6646,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="AKS", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6701,6 +6821,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="GKE", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -6853,6 +6974,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="GKE", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7033,6 +7155,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Nebius", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7140,6 +7263,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Nebius", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7275,6 +7399,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Vultr", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7381,6 +7506,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Vultr", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7515,6 +7641,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Civo", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7633,6 +7760,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Civo", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7779,6 +7907,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Existing", stack="Standard", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -7886,6 +8015,7 @@ def test_compose(case: ComposeCase) -> None: composite=_serving_stack( cloud="Existing", stack="Dynamo", + dra_drivers=["gpu.nvidia.com"], secrets=[ v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), v1alpha1.Secret(type="GoogleApplicationCredentials", name="sa-secret", key="private_key"), @@ -8010,6 +8140,231 @@ def test_compose(case: ComposeCase) -> None: "usage-modelexpress-crds-modelmetadatas.modelexpress.nvidia.com-by-modelexpress-server", }, ), + ComposedResourceKeysCase( + name="ExistingStandardCPUOnly", + reason=( + "With draDrivers [dra.cpu], only the CPU device stack joins the untagged components: " + "dra-driver-cpu and its critical-pods quota install, and the NVIDIA and AMD stacks - " + "NFD included - stay off the cluster." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_serving_stack( + cloud="Existing", + stack="Standard", + dra_drivers=["dra.cpu"], + secrets=[ + v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), + ], + gateway=v1alpha1.Gateway( + hostname="test-backend.gateways.example.com", + clientCAs=[ + v1alpha1.ClientCA( + name="eu", + certificate="-----BEGIN CERTIFICATE-----\nfleet\n-----END CERTIFICATE-----\n", + ) + ], + ), + gpu=None, + ), + resources={ + "provider-config-kubernetes": _observed_ready(), + "provider-config-helm": _observed_ready(), + "gateway": _observed_ready(), + "gateway-class": _observed_ready(), + "gateway-ca-certificate": _observed_ready(), + "gateway-ca-issuer": _observed_ready(), + "gateway-serving-certificate": _observed_ready(), + "gateway-ca-bundle": _observed_ready(), + "gateway-ca-configmap": _observed_ready(), + "gateway-client-ca-bundle": _observed_ready(), + "gateway-client-auth": _observed_ready(), + "usage-gateway-class-by-gateway": _observed_ready(), + "usage-envoy-gateway-by-gateway-class": _observed_ready(), + "ai-gateway": _observed_ready(), + "ai-gateway-crds": _observed_ready(), + "dra-driver-cpu": _observed_ready(), + "dra-driver-cpu-critical-pods-quota": _observed_ready(), + "envoy-gateway": _observed_ready(), + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.x-k8s.io": _observed_ready(), + "gateway-namespace": _observed_ready(), + "gateway-proxy": _observed_ready(), + "gateway-selfsigned-issuer": _observed_ready(), + "trust-manager": _observed_ready(), + "usage-ai-gateway-crds-by-ai-gateway": _observed_ready(), + "usage-cert-manager-by-envoy-gateway": _observed_ready(), + "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), + "cert-manager": _observed_ready(), + "kube-prometheus-stack": _observed_ready(), + "leader-worker-set": _observed_ready(), + }, + ), + ), + want={ + # Every cloud and stack. + "provider-config-kubernetes", + "provider-config-helm", + "gateway", + "gateway-class", + "gateway-ca-certificate", + "gateway-ca-issuer", + "gateway-serving-certificate", + "gateway-ca-bundle", + "gateway-ca-configmap", + "gateway-client-ca-bundle", + "gateway-client-auth", + "usage-gateway-class-by-gateway", + "usage-envoy-gateway-by-gateway-class", + # The common components, including the dra.cpu-tagged device + # stack this cluster's classes claim. + "ai-gateway", + "ai-gateway-crds", + "dra-driver-cpu", + "dra-driver-cpu-critical-pods-quota", + "envoy-gateway", + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", + "gaie-crds-inferencepools.inference.networking.k8s.io", + "gaie-crds-inferencepools.inference.networking.x-k8s.io", + "gateway-namespace", + "gateway-proxy", + "gateway-selfsigned-issuer", + "trust-manager", + "usage-ai-gateway-crds-by-ai-gateway", + "usage-cert-manager-by-envoy-gateway", + "usage-cert-manager-by-gateway-selfsigned-issuer", + "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-gateway-selfsigned-issuer", + "usage-gateway-selfsigned-issuer-by-trust-manager", + # The hand-written Existing half, less its GPU stacks. + "cert-manager", + "kube-prometheus-stack", + # The Standard stack. + "leader-worker-set", + }, + ), + ComposedResourceKeysCase( + name="ExistingStandardUnfiltered", + reason=( + "With draDrivers absent - a ServingStack from before the field existed - nothing filters: " + "every device stack installs, NVIDIA, AMD and CPU alike, so an upgrade doesn't tear down " + "a running stack before the cluster composition stamps the list." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_serving_stack( + cloud="Existing", + stack="Standard", + dra_drivers=None, + secrets=[ + v1alpha1.Secret(type="Kubeconfig", name="kube-secret", key="kubeconfig"), + ], + gateway=v1alpha1.Gateway( + hostname="test-backend.gateways.example.com", + clientCAs=[ + v1alpha1.ClientCA( + name="eu", + certificate="-----BEGIN CERTIFICATE-----\nfleet\n-----END CERTIFICATE-----\n", + ) + ], + ), + gpu=None, + ), + resources={ + "provider-config-kubernetes": _observed_ready(), + "provider-config-helm": _observed_ready(), + "gateway": _observed_ready(), + "gateway-class": _observed_ready(), + "gateway-ca-certificate": _observed_ready(), + "gateway-ca-issuer": _observed_ready(), + "gateway-serving-certificate": _observed_ready(), + "gateway-ca-bundle": _observed_ready(), + "gateway-ca-configmap": _observed_ready(), + "gateway-client-ca-bundle": _observed_ready(), + "gateway-client-auth": _observed_ready(), + "usage-gateway-class-by-gateway": _observed_ready(), + "usage-envoy-gateway-by-gateway-class": _observed_ready(), + "ai-gateway": _observed_ready(), + "ai-gateway-crds": _observed_ready(), + "amd-gpu-operator": _observed_ready(), + "dra-driver-cpu": _observed_ready(), + "dra-driver-cpu-critical-pods-quota": _observed_ready(), + "dra-driver-critical-pods-quota": _observed_ready(), + "envoy-gateway": _observed_ready(), + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.k8s.io": _observed_ready(), + "gaie-crds-inferencepools.inference.networking.x-k8s.io": _observed_ready(), + "gateway-namespace": _observed_ready(), + "gateway-proxy": _observed_ready(), + "gateway-selfsigned-issuer": _observed_ready(), + "trust-manager": _observed_ready(), + "usage-ai-gateway-crds-by-ai-gateway": _observed_ready(), + "usage-cert-manager-by-amd-gpu-operator": _observed_ready(), + "usage-cert-manager-by-envoy-gateway": _observed_ready(), + "usage-cert-manager-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-namespace-by-gateway-proxy": _observed_ready(), + "usage-gateway-namespace-by-gateway-selfsigned-issuer": _observed_ready(), + "usage-gateway-selfsigned-issuer-by-trust-manager": _observed_ready(), + "usage-node-feature-discovery-by-amd-gpu-operator": _observed_ready(), + "cert-manager": _observed_ready(), + "kube-prometheus-stack": _observed_ready(), + "node-feature-discovery": _observed_ready(), + "nvidia-dra-driver-gpu": _observed_ready(), + "leader-worker-set": _observed_ready(), + }, + ), + ), + want={ + # Every cloud and stack. + "provider-config-kubernetes", + "provider-config-helm", + "gateway", + "gateway-class", + "gateway-ca-certificate", + "gateway-ca-issuer", + "gateway-serving-certificate", + "gateway-ca-bundle", + "gateway-ca-configmap", + "gateway-client-ca-bundle", + "gateway-client-auth", + "usage-gateway-class-by-gateway", + "usage-envoy-gateway-by-gateway-class", + # The common components, device stacks included. + "ai-gateway", + "ai-gateway-crds", + "dra-driver-cpu", + "dra-driver-cpu-critical-pods-quota", + "dra-driver-critical-pods-quota", + "envoy-gateway", + "gaie-crds-inferenceobjectives.inference.networking.x-k8s.io", + "gaie-crds-inferencepools.inference.networking.k8s.io", + "gaie-crds-inferencepools.inference.networking.x-k8s.io", + "gateway-namespace", + "gateway-proxy", + "gateway-selfsigned-issuer", + "trust-manager", + "usage-ai-gateway-crds-by-ai-gateway", + "usage-cert-manager-by-envoy-gateway", + "usage-cert-manager-by-gateway-selfsigned-issuer", + "usage-gateway-namespace-by-gateway-proxy", + "usage-gateway-namespace-by-gateway-selfsigned-issuer", + "usage-gateway-selfsigned-issuer-by-trust-manager", + # The hand-written Existing half, both GPU stacks included. + "amd-gpu-operator", + "cert-manager", + "kube-prometheus-stack", + "node-feature-discovery", + "nvidia-dra-driver-gpu", + "usage-cert-manager-by-amd-gpu-operator", + "usage-node-feature-discovery-by-amd-gpu-operator", + # The Standard stack. + "leader-worker-set", + }, + ), ] diff --git a/functions/compose-serving-stack/tests/test_stacks.py b/functions/compose-serving-stack/tests/test_stacks.py index 0570a6320..bbcc1c86d 100644 --- a/functions/compose-serving-stack/tests/test_stacks.py +++ b/functions/compose-serving-stack/tests/test_stacks.py @@ -45,6 +45,16 @@ class JoinFailsClosedCase: want: str +@dataclasses.dataclass +class SupportedDriversCase: + """A test case for stacks.supported_drivers.""" + + name: str + reason: str + cloud: str + want: frozenset[str] + + @dataclasses.dataclass class WithNvLinkDisabledCase: """A test case for civo.with_nvlink_disabled.""" @@ -190,6 +200,178 @@ def test_join_fails_closed(case: JoinFailsClosedCase) -> None: stacks.join(case.cloud, case.stack) # ty: ignore[invalid-argument-type] # cases pass values outside the Literals +# The drivers each cloud's serving stack supports, pinned as literals so +# a new dra_drivers tag or external declaration lands here deliberately - +# and moves compose-inference-cluster's hand-kept _SUPPORTED_DRA_DRIVERS +# mirror in the same review. Existing additionally treats any driver +# beyond its tags as externally installed (see test_join_unknown_existing). +SUPPORTED_DRIVERS_CASES = [ + SupportedDriversCase( + name="EKS", + reason="EKS tags the NVIDIA and CPU stacks and declares dra.net installed outside the serving stack.", + cloud="EKS", + want=frozenset({"gpu.nvidia.com", "dra.cpu", "dra.net"}), + ), + SupportedDriversCase( + name="AKS", + reason="AKS tags the NVIDIA and CPU stacks.", + cloud="AKS", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="GKE", + reason="GKE tags the NVIDIA and CPU stacks.", + cloud="GKE", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Nebius", + reason="Nebius tags the NVIDIA and CPU stacks.", + cloud="Nebius", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Vultr", + reason="Vultr tags the NVIDIA and CPU stacks.", + cloud="Vultr", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Civo", + reason="Civo tags the NVIDIA and CPU stacks.", + cloud="Civo", + want=frozenset({"gpu.nvidia.com", "dra.cpu"}), + ), + SupportedDriversCase( + name="Existing", + reason="Existing is the one cloud that tags the AMD stack, beside NVIDIA and CPU.", + cloud="Existing", + want=frozenset({"gpu.nvidia.com", "gpu.amd.com", "dra.cpu"}), + ), +] + + +@pytest.mark.parametrize("case", SUPPORTED_DRIVERS_CASES, ids=lambda case: case.name) +def test_supported_drivers(case: SupportedDriversCase) -> None: + """Each cloud's derived supported drivers match the pinned set.""" + assert stacks.supported_drivers(case.cloud) == case.want, case.reason # ty: ignore[invalid-argument-type] # parametrize erases the Literal + + +@pytest.mark.parametrize("stack", stacks.stacks()) +@pytest.mark.parametrize("cloud", stacks.clouds()) +def test_join_filter_empty(cloud: stacks.Cloud, stack: stacks.Stack) -> None: + """An empty draDrivers list keeps exactly the untagged components.""" + assert stacks.join(cloud, stack, dra_drivers=[]) == [c for c in stacks.join(cloud, stack) if not c.dra_drivers] + + +@pytest.mark.parametrize("stack", stacks.stacks()) +@pytest.mark.parametrize("cloud", stacks.clouds()) +def test_join_filter_all(cloud: stacks.Cloud, stack: stacks.Stack) -> None: + """Listing every tagged driver reproduces the unfiltered join.""" + joined = stacks.join(cloud, stack) + drivers = sorted({d for c in joined for d in c.dra_drivers}) + assert stacks.join(cloud, stack, dra_drivers=drivers) == joined + + +def test_join_filter_nvidia() -> None: + """gpu.nvidia.com alone installs the NVIDIA stack and no other.""" + keys = {c.key for c in stacks.join("Existing", "Standard", dra_drivers=["gpu.nvidia.com"])} + assert {"node-feature-discovery", "nvidia-dra-driver-gpu", "dra-driver-critical-pods-quota"} <= keys + assert {"dra-driver-cpu", "dra-driver-cpu-critical-pods-quota", "amd-gpu-operator"}.isdisjoint(keys) + + +def test_join_filter_cpu() -> None: + """dra.cpu alone installs the CPU driver and neither GPU stack - NFD included.""" + keys = {c.key for c in stacks.join("Existing", "Standard", dra_drivers=["dra.cpu"])} + assert {"dra-driver-cpu", "dra-driver-cpu-critical-pods-quota"} <= keys + assert {"node-feature-discovery", "nvidia-dra-driver-gpu", "amd-gpu-operator"}.isdisjoint(keys) + + +def test_join_filter_amd() -> None: + """gpu.amd.com on Existing installs the AMD operator and the NFD both GPU stacks share.""" + keys = {c.key for c in stacks.join("Existing", "Standard", dra_drivers=["gpu.amd.com"])} + assert {"amd-gpu-operator", "node-feature-discovery"} <= keys + assert {"nvidia-dra-driver-gpu", "dra-driver-critical-pods-quota", "dra-driver-cpu"}.isdisjoint(keys) + + +@pytest.mark.parametrize("cloud", [c for c in stacks.clouds() if c != "Existing"]) +def test_join_unknown_driver(cloud: stacks.Cloud) -> None: + """A driver the cloud neither tags nor declares external fails the join.""" + with pytest.raises(ValueError, match=r"gpu\.example\.com"): + stacks.join(cloud, "Standard", dra_drivers=["gpu.example.com"]) + + +def test_join_unknown_existing() -> None: + """Existing treats an unlisted driver as externally installed: no raise, no tagged component.""" + joined = stacks.join("Existing", "Standard", dra_drivers=["gpu.example.com"]) + assert [c.key for c in joined if c.dra_drivers] == [] + + +def test_join_external_driver() -> None: + """EKS's declared-external dra.net is accepted and installs nothing; elsewhere it raises.""" + joined = stacks.join("EKS", "Standard", dra_drivers=["dra.net"]) + assert [c.key for c in joined if c.dra_drivers] == [] + with pytest.raises(ValueError, match=r"dra\.net"): + stacks.join("GKE", "Standard", dra_drivers=["dra.net"]) + + +def test_join_tagged_dependency(monkeypatch: pytest.MonkeyPatch) -> None: + """A dependency not carrying every driver its dependent does fails every join for the cloud.""" + broken = [ + stacks.Chart( + key="dep", + release="mp-dep", + namespace="d", + chart="dep", + repository="r", + version="1", + wait=True, + dra_drivers=["gpu.nvidia.com"], + ), + stacks.Chart( + key="user", + release="mp-user", + namespace="d", + chart="user", + repository="r", + version="1", + depends_on=["dep"], + dra_drivers=["gpu.nvidia.com", "dra.cpu"], + ), + ] + monkeypatch.setitem(stacks._CLOUDS, "Existing", broken) + with pytest.raises(ValueError, match="filtering could remove"): + stacks.join("Existing", "Standard") + + +def test_join_untagged_dependent(monkeypatch: pytest.MonkeyPatch) -> None: + """An untagged component depending on a tagged one fails: the filter keeps it but can drop its dependency.""" + broken = [ + stacks.Chart( + key="dep", + release="mp-dep", + namespace="d", + chart="dep", + repository="r", + version="1", + wait=True, + dra_drivers=["gpu.nvidia.com"], + ), + stacks.Chart( + key="user", + release="mp-user", + namespace="d", + chart="user", + repository="r", + version="1", + depends_on=["dep"], + ), + ] + monkeypatch.setitem(stacks._CLOUDS, "Existing", broken) + with pytest.raises(ValueError, match="filtering could remove"): + stacks.join("Existing", "Standard") + + @pytest.mark.parametrize("cloud", ["AKS", "EKS", "GKE"]) def test_generated_dcgm_csv(cloud: str) -> None: """Each generated cloud's inline DCGM exporter CSV is the shared one. @@ -305,6 +487,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: version="v26.3.3", wait=True, depends_on=["node-feature-discovery", "cert-manager"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -354,6 +537,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", depends_on=["gpu-operator", "nvlink-disabled-driver-h100-pool"], + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "nvidiaDriverRoot": "/run/nvidia/driver", @@ -413,6 +597,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: version="v26.3.3", wait=True, depends_on=["node-feature-discovery", "cert-manager"], + dra_drivers=["gpu.nvidia.com"], values={ "ccManager": {"enabled": False}, "cdi": {"default": True, "enabled": True}, @@ -459,6 +644,7 @@ def test_generated_dcgm_csv(cloud: str) -> None: repository="oci://registry.k8s.io/dra-driver-nvidia/charts", version="0.4.1", depends_on=["gpu-operator", "nvlink-disabled-driver-pool-a", "nvlink-disabled-driver-pool-b"], + dra_drivers=["gpu.nvidia.com"], values={ "gpuResourcesEnabledOverride": True, "nvidiaDriverRoot": "/run/nvidia/driver", diff --git a/functions/compose-vultr-cluster/function/fn.py b/functions/compose-vultr-cluster/function/fn.py index 830a0a4f4..8db939403 100644 --- a/functions/compose-vultr-cluster/function/fn.py +++ b/functions/compose-vultr-cluster/function/fn.py @@ -85,6 +85,12 @@ _GPU_TAINT_VALUE = "true" _GPU_TAINT_EFFECT = "NoSchedule" +# Label and taint on CPU inference pools. The label is what the serving +# stack's dra-driver-cpu selects its nodes by; the taint keeps system +# workloads off the pool as the GPU taint does on GPU pools, and engine +# pods tolerate it (compose-model-replica). +_LABEL_CPU = "modelplane.ai/cpu" + # Vultr pre-installs the NVIDIA GPU Operator on every VKE GPU cluster. The # operator's validator DaemonSet runs a four-stage init sequence (driver → # CUDA → device-plugin → full validation). When all pods in the DaemonSet @@ -160,7 +166,12 @@ def compose(self) -> None: if self._cluster_ready() or self._dependents_observed(): self.compose_node_pools() self.compose_provider_config() - self.compose_gpu_observer() + # Vultr pre-installs the NVIDIA GPU Operator only on clusters + # with GPU pools, so a CPU-only cluster has no validator + # DaemonSet to observe - composing the observer anyway would + # hold the cluster un-Ready forever. + if any(pool.role == "GPU" for pool in self.xr.spec.nodePools): + self.compose_gpu_observer() self.write_status() self.mark_readiness() @@ -290,6 +301,8 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> vkenodepoolv1beta1.KubernetesNo if pool.role == "GPU" and pool.gpu: labels.append(vkenodepoolv1beta1.Label(key=_LABEL_GPU, value=pool.gpu.acceleratorType)) labels.append(vkenodepoolv1beta1.Label(key=_LABEL_DEVICE_PLUGIN_KEY, value=_LABEL_DEVICE_PLUGIN_VALUE)) + elif pool.role == "CPU": + labels.append(vkenodepoolv1beta1.Label(key=_LABEL_CPU, value="true")) fp = vkenodepoolv1beta1.ForProvider( label=pool.name, @@ -309,6 +322,14 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> vkenodepoolv1beta1.KubernetesNo effect=_GPU_TAINT_EFFECT, ), ] + elif pool.role == "CPU": + fp.taints = [ + vkenodepoolv1beta1.Taint( + key=_LABEL_CPU, + value="true", + effect=_GPU_TAINT_EFFECT, + ), + ] # maxNodeCount opts into VKE's server-side autoscaling. if pool.maxNodeCount is not None: diff --git a/functions/compose-vultr-cluster/tests/test_fn.py b/functions/compose-vultr-cluster/tests/test_fn.py index c6021b0bf..7e735694c 100644 --- a/functions/compose-vultr-cluster/tests/test_fn.py +++ b/functions/compose-vultr-cluster/tests/test_fn.py @@ -179,6 +179,36 @@ def _node_pool_gpu(*, node_quantity: int, autoscaling: dict | None, ready: fnv1. ) +def _node_pool_cpu(*, node_quantity: int, autoscaling: dict | None, ready: fnv1.Ready) -> fnv1.Resource: + """The composed KubernetesNodePool for the cpu-pool pool, with the CPU label and taint and no GPU labels.""" + for_provider: dict[str, Any] = { + "label": "cpu-pool", + "plan": "vc2-6c-16gb", + "nodeQuantity": node_quantity, + "labels": [ + {"key": "modelplane.ai/pool", "value": "cpu-pool"}, + {"key": "modelplane.ai/cpu", "value": "true"}, + ], + "clusterIdSelector": {"matchControllerRef": True}, + "taints": [{"key": "modelplane.ai/cpu", "value": "true", "effect": "NoSchedule"}], + } + if autoscaling is not None: + for_provider.update(autoscaling) + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "vke.vultr.m.upbound.io/v1beta1", + "kind": "KubernetesNodePool", + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": for_provider, + }, + } + ), + ready=ready, + ) + + def _provider_config() -> fnv1.Resource: """The composed provider-kubernetes ProviderConfig for the cluster, which is always ready.""" return fnv1.Resource( @@ -319,6 +349,47 @@ def _to_dict(msg: message.Message) -> dict: context=structpb.Struct(), ), ), + Case( + name="ClusterReadyCPUPool", + reason=( + "A CPU-role pool composes with the modelplane.ai/pool and modelplane.ai/cpu labels and the CPU taint, " + "no GPU or device-plugin labels, and no GPU observer - Vultr pre-installs the GPU operator only on " + "clusters with GPU pools, so there is no validator DaemonSet to observe." + ), + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + plan="vc2-6c-16gb", + maxNodeCount=4, + ), + ], + ), + resources={ + "cluster": _observed_cluster(ready=True), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(), + resources={ + "cluster": _cluster(ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": _node_pool_cpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-kubernetes": _provider_config(), + }, + ), + context=structpb.Struct(), + ), + ), Case( name="ClusterReadyRegressed", reason="With the cluster's Ready condition regressed to False but its ProviderConfig observed, a VultrCluster keeps the node pool, ProviderConfig and GPU observer composed.", @@ -508,7 +579,10 @@ def _to_dict(msg: message.Message) -> dict: # which of them sets the autoscaler's floor. Case( name="SystemRolePool", - reason="A VultrCluster's System-role pool autoscales from a floor of 2 nodes to maxNodeCount, and gets neither GPU labels nor a taint.", + reason=( + "A VultrCluster's System-role pool autoscales from a floor of 2 nodes to maxNodeCount, and gets " + "neither GPU labels nor a taint - and with no GPU pool there is no GPU observer to compose." + ), req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr( @@ -556,7 +630,6 @@ def _to_dict(msg: message.Message) -> dict: ), ), "provider-config-kubernetes": _provider_config(), - "gpu-observer": _gpu_observer(ready=fnv1.READY_UNSPECIFIED), }, ), context=structpb.Struct(), diff --git a/schemas/.lock.json b/schemas/.lock.json index ad2bd8e12..01adac440 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "1e73064210fec7fa5508d4af954e85d2adba93ccb40101324d926f950e8d2e2f", + "fs://apis": "cb6d31114c84ee9720f2dcd68e9eaa36227e2376f3399200723d86b305cdb4a1", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", "xpkg://xpkg.upbound.io/upbound/provider-aws-efs:v2.8.1": "sha256:ebb1bcd8dc9a7e60e97a324652fbd9d1b0609ddbb1107db518ce999cbc1113bd", diff --git a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py index 8fadaeaea..44953b0b0 100644 --- a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py @@ -99,9 +99,9 @@ class Accelerator(BaseModel): class Aks(BaseModel): - accelerator: Accelerator + accelerator: Accelerator | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Omit for a pool with no GPU, which is composed with role CPU: a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 vmSize: constr(min_length=1) @@ -119,9 +119,9 @@ class AcceleratorModel(BaseModel): class Civo(BaseModel): - accelerator: AcceleratorModel + accelerator: AcceleratorModel | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Omit for a pool with no GPU, which is composed with role CPU: a standard node image and the modelplane.ai/cpu label and taint. """ size: constr(min_length=1, max_length=63) """ @@ -138,9 +138,9 @@ class AcceleratorModel1(BaseModel): class Eks(BaseModel): - accelerator: AcceleratorModel1 + accelerator: AcceleratorModel1 | None = None """ - GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Omit for a group with no GPU, which is composed with role CPU: a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 instanceType: constr(min_length=1) @@ -158,9 +158,9 @@ class AcceleratorModel2(BaseModel): class Gke(BaseModel): - accelerator: AcceleratorModel2 + accelerator: AcceleratorModel2 | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in devices. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block, so count here is the GCP machine's GPU count and need not be restated in devices. Omit for a pool with no GPU, which is composed with role CPU: a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 machineType: constr(min_length=1) @@ -175,9 +175,9 @@ class AcceleratorModel3(BaseModel): class Nebius(BaseModel): - accelerator: AcceleratorModel3 + accelerator: AcceleratorModel3 | None = None """ - GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node group. Provisioning input only: the scheduler matches against spec.devices, not this block. Omit for a group with no GPU, which is composed with role CPU: a standard node image and the modelplane.ai/cpu label and taint. """ diskSizeGb: conint(ge=10) | None = 100 driversPreset: constr(min_length=1, max_length=63) | None = 'cuda13.0' @@ -203,9 +203,9 @@ class AcceleratorModel4(BaseModel): class Vultr(BaseModel): - accelerator: AcceleratorModel4 + accelerator: AcceleratorModel4 | None = None """ - GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. + GPU accelerator to attach when provisioning the node pool. Provisioning input only: the scheduler matches against spec.devices, not this block. Omit for a pool with no GPU, which is composed with role CPU: a standard node image and the modelplane.ai/cpu label and taint. """ plan: constr(min_length=1, max_length=63) """ diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index da447c98f..e97ff37b4 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -336,6 +336,13 @@ class Gateway(BaseModel): """ +class ProviderConfigRef(BaseModel): + name: str | None = None + """ + Name of the ProviderConfig targeting the remote cluster. Used by ModelReplica to create resources on the cluster. + """ + + class Attributes(BaseModel): bool_: bool | None = Field(None, alias='bool') int_: int | None = Field(None, alias='int') @@ -357,7 +364,7 @@ class Device(BaseModel): name: constr(max_length=63) -class GpuPool(BaseModel): +class SchedulablePool(BaseModel): devices: list[Device] | None = Field(None, max_length=16) """ Devices copied from the pool's InferenceClass. ModelDeployment.nodeSelector matches against these. @@ -372,13 +379,6 @@ class GpuPool(BaseModel): """ -class ProviderConfigRef(BaseModel): - name: str | None = None - """ - Name of the ProviderConfig targeting the remote cluster. Used by ModelReplica to create resources on the cluster. - """ - - class Status(BaseModel): cache: CacheModel | None = None """ @@ -392,15 +392,15 @@ class Status(BaseModel): """ This cluster's own gateway: the Envoy gateway the serving stack runs at the cluster's edge, which an InferenceGateway routes to. These fields are how an InferenceGateway reaches it. """ - gpuPools: list[GpuPool] | None = Field(None, max_length=8) - """ - Schedulable GPU node pools on this cluster, derived from the referenced classes and the per-pool node counts. ModelDeployment scheduling matches against these. - """ namespace: str | None = None """ Namespace where the internal XRs (cluster, backend) were created. """ providerConfigRef: ProviderConfigRef | None = None + schedulablePools: list[SchedulablePool] | None = Field(None, max_length=8) + """ + Schedulable node pools on this cluster - GPU and CPU alike; every pool whose class declares devices - derived from the referenced classes and the per-pool node counts. ModelDeployment scheduling matches against these. + """ class InferenceCluster(BaseModel): diff --git a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py index f75e5aedb..95ef71792 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py @@ -99,9 +99,9 @@ class NodePool(BaseModel): """ Initial number of nodes. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and get the NVIDIA driver installed by AKS. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and get the NVIDIA driver installed by AKS. CPU pools host CPU inference workloads: no GPU driver, and the modelplane.ai/cpu label and taint. """ vmSize: constr(min_length=1, max_length=63) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py index d4ce66e24..3c2386788 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py @@ -76,9 +76,9 @@ class NodePool(BaseModel): """ Number of nodes. Fixed unless maxNodeCount enables autoscaling. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. CPU pools host CPU inference workloads: a non-GPU size and the modelplane.ai/cpu label and taint. """ size: constr(min_length=1, max_length=63) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py index 47489cab0..192f72dea 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py @@ -125,9 +125,9 @@ class NodePool(BaseModel): """ Initial number of nodes. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and use a GPU-enabled AMI. + Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and use a GPU-enabled AMI. CPU groups host CPU inference workloads: a standard AMI, no GPU, and the modelplane.ai/cpu label and taint. """ zones: list[Zone] | None = Field(None, max_length=8, min_length=1) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py index fab5c4dfd..7a408a0d7 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py @@ -107,9 +107,9 @@ class NodePool(BaseModel): """ Initial number of nodes. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. CPU pools host CPU inference workloads: a standard node image, no GPU, and the modelplane.ai/cpu label and taint. """ zones: list[Zone] | None = Field(None, max_length=8, min_length=1) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py index 013c65d34..4e44a948e 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py @@ -114,9 +114,9 @@ class NodePool(BaseModel): """ Resource preset within the platform (e.g. 8gpu-128vcpu-1600gb, 4vcpu-16gb). Determines the GPU, vCPU, and memory shape of each node. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this group runs. System groups host controllers, gateways, and infrastructure. GPU groups host inference workloads and are tainted to exclude non-GPU pods. CPU groups host CPU inference workloads: a CPU platform, no GPU, and the modelplane.ai/cpu label and taint. """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py index c7aab827a..1633ac1cd 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py @@ -5,7 +5,7 @@ from typing import Literal -from pydantic import AwareDatetime, BaseModel, Field, constr +from pydantic import AwareDatetime, BaseModel, Field, RootModel, constr from .....io.k8s.apimachinery.pkg.apis.meta import v1 @@ -41,6 +41,10 @@ class Crossplane(BaseModel): resourceRefs: list[ResourceRef] | None = None +class DraDriver(RootModel[constr(min_length=1, max_length=253)]): + root: constr(min_length=1, max_length=253) + + class ClientCA(BaseModel): certificate: constr(min_length=1, max_length=16384) """ @@ -119,6 +123,10 @@ class Spec(BaseModel): """ Configures how Crossplane will reconcile this composite resource """ + draDrivers: list[DraDriver] | None = Field(None, max_length=16) + """ + DRA driver names whose device stacks this cluster needs (gpu.nvidia.com, dra.cpu, ...), collected by the cluster composition from the claim: DRA devices of the cluster's referenced InferenceClasses. A component tagged with a driver installs only when that driver is listed; untagged components always install. When the field is absent no filtering happens and every component installs, so a ServingStack from before the field existed keeps its stack until the cluster composition stamps the list. + """ gateway: Gateway """ Configuration for the cluster's inference traffic gateway. diff --git a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py index 4e312564d..be03c5199 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py @@ -80,9 +80,9 @@ class NodePool(BaseModel): """ Vultr plan ID for the pool's nodes (e.g. vcg-l40s-16c-180g-48vram, vc2-6c-16gb). This is the instance type used by other clouds; list plans with vultr-cli plans list. """ - role: Literal['System', 'GPU'] + role: Literal['System', 'GPU', 'CPU'] """ - Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. + Determines what workloads this pool runs. System pools host controllers, gateways, and infrastructure. GPU pools host inference workloads and are tainted to exclude non-GPU pods. CPU pools host CPU inference workloads: a non-GPU plan and the modelplane.ai/cpu label and taint. """ diff --git a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py index 68d2b0b87..c1632636b 100644 --- a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py @@ -236,7 +236,7 @@ class Worker(BaseModel): class Member(BaseModel): nodeSelector: NodeSelector | None = None """ - The per-node device request for this member's pods: what devices each pod needs from its node. The scheduler matches it against a candidate pool's InferenceClass devices (surfaced on InferenceCluster status.gpuPools) and places the member on a pool that satisfies it, preferring one pool for the whole engine and splitting members across pools only when no pool satisfies them all. claim: DRA requests also become DeviceRequests in the ResourceClaim the member's pods bind GPUs through. A GPU request's count is the GPUs per node. Omitted, the member claims no devices and schedules onto its engine's pool - a coordinator-only leader. At least one member per engine must carry a nodeSelector, and at least one member's requests must resolve to a claimable (claim: DRA) device; an engine that matches only synthetic devices leaves its pods nothing to claim, so the scheduler treats such a pool as ineligible and the deployment reports InsufficientCapacity. + The per-node device request for this member's pods: what devices each pod needs from its node. The scheduler matches it against a candidate pool's InferenceClass devices (surfaced on InferenceCluster status.schedulablePools) and places the member on a pool that satisfies it, preferring one pool for the whole engine and splitting members across pools only when no pool satisfies them all. claim: DRA requests also become DeviceRequests in the ResourceClaim the member's pods bind GPUs through. A GPU request's count is the GPUs per node. Omitted, the member claims no devices and schedules onto its engine's pool - a coordinator-only leader. At least one member per engine must carry a nodeSelector, and at least one member's requests must resolve to a claimable (claim: DRA) device; an engine that matches only synthetic devices leaves its pods nothing to claim, so the scheduler treats such a pool as ineligible and the deployment reports InsufficientCapacity. """ role: Literal['Standalone', 'Leader', 'Worker'] | None = 'Standalone' """ From 75569902cfac648c133889f2bb06b57d576e18bd Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 22 Oct 2026 14:53:39 +0200 Subject: [PATCH 2/4] test(e2e): update e2e test to support cpu, nvidia and amd Signed-off-by: Christopher Haar --- e2e/README.md | 46 ++-- e2e/clouds/__init__.py | 199 ++++++++++++++---- e2e/clouds/test_placement.py | 135 ++++++++---- e2e/environment.py | 13 +- e2e/test_legs.py | 24 +++ .../tests/test_backends.py | 24 ++- .../function/stacks/common.py | 18 +- 7 files changed, 352 insertions(+), 107 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index ea676ce1b..8311e8461 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -96,21 +96,27 @@ cloud's serving stack — the AICR-generated component lists for EKS/AKS/GKE, or the hand-written halves for Civo, Nebius, Vultr and Existing — onto the same workload kind cluster, with no cloud credentials. The trick is [KWOK](https://kwok.sigs.k8s.io): bring-up applies its pinned `kwok.yaml` and -`stage-fast.yaml` manifests and two fake Nodes labelled and tainted exactly -like that cloud's GPU pools. The real kind node plays the untainted system -pool, so system components install and run for real; the GPU-toleration -DaemonSets fan out to the fake nodes, where KWOK marks their pods Running — -which is all helm `--wait` and the composition's readiness gates need. +`stage-fast.yaml` manifests and two fake Nodes per device pool, labelled and +tainted exactly like that cloud's pools — an NVIDIA GPU pool and a CPU pool +on every leg, plus an AMD GPU pool on Existing, the one cloud that installs +the AMD stack. The real kind node plays the untainted system pool, so system +components install and run for real; each device stack's DaemonSets fan out +to their own pool's fake nodes, where KWOK marks their pods Running — which +is all helm `--wait` and the composition's readiness gates need. The clouds are one data-driven matrix: each leg in `clouds/__init__.py` holds -everything cloud-specific — the fake nodes' labels, the GPU taint value, the -stack's `spec.cloud`, and the DaemonSets expected to fan out to the GPU pool. -Adding fidelity for a cloud means editing its leg, not a copy of the test. +everything cloud-specific — each fake pool's labels, the GPU taint value, the +stack's `spec.cloud`, the `spec.draDrivers` its classes would claim (which +filters the stack to exactly those device stacks), and the DaemonSets +expected to fan out to the pools. Adding fidelity for a cloud means editing +its leg, not a copy of the test. A leg is itself a copy of what its cloud's cluster composition applies, so it can drift from the composition as well as the stack from it. `e2e/test_legs.py` pins each leg's taint value to the `_GPU_TAINT*` constants of the composition -it mirrors (GKE's managed taint and BYO pools' come from outside Modelplane). +it mirrors (GKE's managed taint and BYO pools' come from outside Modelplane), +and the shared CPU taint key to every composition's `_LABEL_CPU` — GKE +included, which sets the CPU taint itself where its GPU taint is managed. It reads the compositions' source and needs no cluster, so it runs as the `e2e-legs` check in `nix flake check` rather than here: a composition-only change triggers the flake checks but not this workflow. The labels are a known @@ -130,15 +136,19 @@ clusters (`--no-apply` stops short of the fake nodes and the XR, `--test` reruns the tests alone). A session fixture waits for the `ServingStack` to go Ready, then the asserts are two-sided: -- **DaemonSet fan-out**: the leg's expected DaemonSets (the DRA kubelet - plugin, NFD worker, node-exporter; toolkit-hardening on AKS) reach their - ready counts — a toleration too *narrow* (e.g. the wrong `Equal` value on - AKS) shows up as a shortfall. -- **Placement/toleration audit**: no pod outside the GPU allowlist carries a - wildcard or `nvidia.com/gpu` toleration, nothing unexpected sits on a fake - GPU node, and GPU-only pods never sit on the system pool — a toleration too - *broad* (the "tolerates every taint" class of bug) fails here even when the - scheduler happened to keep the pod on the system node. +- **DaemonSet fan-out**: the leg's expected DaemonSets (the NVIDIA DRA + kubelet plugin, dra-driver-cpu, NFD worker, node-exporter; the AMD + operands on Existing; toolkit-hardening on AKS) reach their ready counts — + a toleration too *narrow* (e.g. the wrong `Equal` value on AKS), a + selector missing its pool, or a filter that wrongly dropped a claimed + stack shows up as a shortfall. +- **Placement/toleration audit**: no pod outside a pool's allowlist carries + a wildcard or that pool's taint toleration, everything on a fake node + belongs to that node's pool, and pool-only pods (the DRA drivers, the GPU + operands) never sit anywhere else — a toleration too *broad* (the + "tolerates every taint" class of bug) fails here even when the scheduler + happened to keep the pod on the system node, and so does a device stack + installing where its driver was never claimed. What cloud mode does **not** test: AKS's admission rejection of wildcard tolerations (the audit approximates it client-side), real DRA allocation and diff --git a/e2e/clouds/__init__.py b/e2e/clouds/__init__.py index b4314a5fc..66dfa6ac6 100644 --- a/e2e/clouds/__init__.py +++ b/e2e/clouds/__init__.py @@ -14,9 +14,15 @@ """The cloud legs of the serving-stack e2e matrix. See e2e/README.md. -Each leg shapes two fake, KWOK-managed GPU Nodes the way its cloud's -cluster composition (or platform) shapes real ones - labels and taint - -and names the DaemonSets the serving stack must fan out to them. +Each leg shapes fake, KWOK-managed Nodes the way its cloud's cluster +composition (or platform) shapes real ones - labels and taint - and +names the DaemonSets the serving stack must fan out to them. A leg +carries one pool per device stack its classes would claim: an NVIDIA +GPU pool and a CPU pool on every cloud, plus an AMD GPU pool on +Existing, the one cloud that installs the AMD stack. The ServingStack +under test names those stacks' DRA drivers in spec.draDrivers, the way +compose-inference-cluster stamps them, so the legs exercise the filter: +each device stack must reach its own pool and stay off the others'. Everything cloud-specific lives here, so adding fidelity for a cloud means editing its leg, not a copy of the test. """ @@ -28,19 +34,28 @@ SERVING_STACK = "e2e-cloud" NAMESPACE = "modelplane-system" -# Two fake nodes, so per-node DaemonSet fan-out is visible. +# Two fake nodes per pool, so per-node DaemonSet fan-out is visible. FAKE_NODES = ("kwok-gpu-0", "kwok-gpu-1") +FAKE_CPU_NODES = ("kwok-cpu-0", "kwok-cpu-1") +FAKE_AMD_NODES = ("kwok-amd-0", "kwok-amd-1") + +# The taints the non-NVIDIA pools carry, fixed across clouds: every +# cluster composition taints CPU pools modelplane.ai/cpu=true, and +# amd.com/gpu=true is the convention BYO AMD pools carry (Modelplane +# doesn't provision them; the AMD stack's operands tolerate exactly it). +CPU_TAINT = {"key": "modelplane.ai/cpu", "value": "true", "effect": "NoSchedule"} +AMD_TAINT = {"key": "amd.com/gpu", "value": "true", "effect": "NoSchedule"} @dataclasses.dataclass(frozen=True) class Cloud: - """One cloud's leg: the fake GPU pool's shape, and what must land on it.""" + """One cloud's leg: its fake device pools' shapes, and what must land on them.""" cloud: str """The ServingStack's spec.cloud.""" labels: dict[str, str] - """The cloud's own GPU node labels, stamped on the fake nodes. + """The cloud's own NVIDIA GPU node labels, stamped on the fake GPU nodes. What the cluster composition (or the cloud's platform) puts on a real GPU node, plus the NFD GPU labels (gpu.present, pci-10de.present) NFD's worker @@ -51,17 +66,42 @@ class Cloud: taint_value: str """The value of the nvidia.com/gpu NoSchedule taint the cloud's GPU pools carry.""" + cpu_labels: dict[str, str] + """The CPU pool's node labels, stamped on the fake CPU nodes. + + What the cluster composition puts on a real CPU node: the pool label and + modelplane.ai/cpu, which is both what keeps system workloads traceable + and the selector dra-driver-cpu finds its nodes by, plus the cloud's + instance-type label for fidelity. + """ + + dra_drivers: tuple[str, ...] + """The ServingStack's spec.draDrivers: the DRA drivers this leg's pools claim. + + What compose-inference-cluster would collect from the leg's classes; the + join filters the stack's tagged device stacks to exactly these. + """ + expected_daemonsets: dict[str, int] - """The DaemonSets that must fan out to the GPU pool: namespace/name to ready count. + """The DaemonSets that must fan out to the device pools: namespace/name to ready count. - The GPU-reaching DaemonSets from the stack, by the names that cloud's - charts give them: 2 means the fake GPU nodes only, 3 means every node, - the real one included. + The node-reaching DaemonSets from the stack, by the names that cloud's + charts give them, counted across every node their tolerations and + selectors reach: the fake pools they serve plus the real node where they + run everywhere. """ gpu_workloads: tuple[str, ...] = () """Pod-name patterns allowed on the GPU pool beyond the common GPU_WORKLOADS, such as AKS's toolkit hardening.""" + amd_labels: dict[str, str] | None = None + """The AMD GPU pool's node labels, when the leg has one (Existing only). + + The pool label plus the NFD label (feature.node.kubernetes.io/amd-gpu) + the AMD stack's operands select their nodes by - pre-set like the NVIDIA + NFD labels, because nothing real runs on a KWOK node. + """ + CLOUDS: dict[str, Cloud] = { # The labels mirror what compose-eks-cluster stamps on GPU node groups @@ -76,10 +116,17 @@ class Cloud: "feature.node.kubernetes.io/pci-10de.present": "true", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "m6i.4xlarge", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/nvidia-dra-driver-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/prometheus-node-exporter": 3, + "monitoring/prometheus-node-exporter": 5, }, ), # The taint VALUE is load-bearing here: the AKS stack's accelerated @@ -98,10 +145,17 @@ class Cloud: "feature.node.kubernetes.io/pci-10de.present": "true", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "Standard_D8s_v5", + "modelplane.ai/pool": "cpufake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/nvidia-dra-driver-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/prometheus-node-exporter": 3, + "monitoring/prometheus-node-exporter": 5, "gpu-operator/nvidia-toolkit-hardening": 2, }, gpu_workloads=("^nvidia-toolkit-hardening",), @@ -125,10 +179,17 @@ class Cloud: "feature.node.kubernetes.io/pci-10de.present": "true", }, taint_value="present", + cpu_labels={ + "node.kubernetes.io/instance-type": "e2-standard-8", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/nvidia-dra-driver-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/prometheus-node-exporter": 3, + "monitoring/prometheus-node-exporter": 5, }, ), # The hand-written Civo half (stacks/clouds/civo.py) keeps its charts' @@ -148,10 +209,17 @@ class Cloud: "feature.node.kubernetes.io/pci-10de.present": "true", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "g4s.kube.large", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5, }, gpu_workloads=("^nvidia-driver-daemonset",), ), @@ -169,10 +237,17 @@ class Cloud: "feature.node.kubernetes.io/pci-10de.present": "true", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "cpu-d3", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, + "dra-driver-cpu/mp-dra-driver-cpu": 2, "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5, }, ), # The hand-written Vultr half (stacks/clouds/vultr.py) has no NFD: Vultr @@ -194,9 +269,16 @@ class Cloud: "feature.node.kubernetes.io/pci-10de.present": "true", }, taint_value="true", + cpu_labels={ + "node.kubernetes.io/instance-type": "vc2-6c-16gb", + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "dra-driver-cpu/mp-dra-driver-cpu": 2, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 5, }, ), # The model e2e already installs the Existing stack, but on an untainted @@ -214,43 +296,70 @@ class Cloud: "feature.node.kubernetes.io/pci-10de.present": "true", }, taint_value="true", + cpu_labels={ + "modelplane.ai/pool": "cpu-fake", + "modelplane.ai/cpu": "true", + }, + # Existing is the one cloud that installs the AMD stack, so its leg + # is the one with an AMD pool. The operands are named for the AMD + # chart's default DeviceConfig; their shape was validated live on + # 8x MI325X bare metal (#434). + amd_labels={ + "modelplane.ai/pool": "amd-fake", + "feature.node.kubernetes.io/amd-gpu": "true", + }, + dra_drivers=("dra.cpu", "gpu.amd.com", "gpu.nvidia.com"), expected_daemonsets={ "nvidia-dra-driver/dra-driver-nvidia-gpu-kubelet-plugin": 2, - "node-feature-discovery/mp-node-feature-discovery-worker": 3, - "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 3, + "dra-driver-cpu/mp-dra-driver-cpu": 2, + "kube-amd-gpu/default-dra-driver": 2, + "kube-amd-gpu/default-node-labeller": 2, + "kube-amd-gpu/default-metrics-exporter": 2, + "node-feature-discovery/mp-node-feature-discovery-worker": 5, + "monitoring/mp-kube-prometheus-stack-prometheus-node-exporter": 7, }, ), } -def nodes(cloud: Cloud) -> list[dict[str, typing.Any]]: - """Return the fake GPU Nodes of a cloud's leg, for the workload cluster. +def _node(name: str, labels: dict[str, str], taint: dict[str, str]) -> dict[str, typing.Any]: + """Return one fake Node: a pool's labels, its taint, and KWOK's marks. - The kwok.x-k8s.io/node annotation hands the Nodes to KWOK, which keeps - them Ready and marks every pod scheduled onto them Running. No - nvidia.com/gpu extended resource: the stack is DRA-mode, with the device - plugin disabled everywhere. + The kwok.x-k8s.io/node annotation hands the Node to KWOK, which keeps it + Ready and marks every pod scheduled onto it Running. No extended + resources: the stack is DRA-mode, with every device plugin disabled. """ - return [ - { - "apiVersion": "v1", - "kind": "Node", - "metadata": { - "name": name, - "annotations": {"kwok.x-k8s.io/node": "fake", "node.alpha.kubernetes.io/ttl": "0"}, - # type: kwok is how bring-up and the tests find the fake - # nodes; every cloud's GPU nodes are linux/amd64. - "labels": { - "type": "kwok", - "kubernetes.io/os": "linux", - "kubernetes.io/arch": "amd64", - **cloud.labels, - }, + return { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": name, + "annotations": {"kwok.x-k8s.io/node": "fake", "node.alpha.kubernetes.io/ttl": "0"}, + # type: kwok is how bring-up and the tests find the fake + # nodes; every cloud's inference nodes are linux/amd64. + "labels": { + "type": "kwok", + "kubernetes.io/os": "linux", + "kubernetes.io/arch": "amd64", + **labels, }, - "spec": {"taints": [{"key": "nvidia.com/gpu", "value": cloud.taint_value, "effect": "NoSchedule"}]}, - } - for name in FAKE_NODES - ] + }, + "spec": {"taints": [taint]}, + } + + +def nodes(cloud: Cloud) -> list[dict[str, typing.Any]]: + """Return the fake Nodes of a cloud's leg, for the workload cluster. + + One tainted pool per device stack the leg claims: the cloud's NVIDIA GPU + pool, its CPU pool, and - where the leg declares one - an AMD GPU pool. + """ + gpu_taint = {"key": "nvidia.com/gpu", "value": cloud.taint_value, "effect": "NoSchedule"} + fake = [_node(name, cloud.labels, gpu_taint) for name in FAKE_NODES] + fake += [_node(name, cloud.cpu_labels, CPU_TAINT) for name in FAKE_CPU_NODES] + if cloud.amd_labels is not None: + fake += [_node(name, cloud.amd_labels, AMD_TAINT) for name in FAKE_AMD_NODES] + return fake def serving_stack(cloud: Cloud) -> dict[str, typing.Any]: @@ -272,6 +381,10 @@ def serving_stack(cloud: Cloud) -> dict[str, typing.Any]: "spec": { "cloud": cloud.cloud, "stack": "Standard", + # What compose-inference-cluster would stamp from the leg's + # classes: the join filters the stack's tagged device stacks to + # exactly these, so each pool gets its stack and no other's. + "draDrivers": list(cloud.dra_drivers), "gateway": {"hostname": f"gateway-{SERVING_STACK}.{NAMESPACE}.svc.cluster.local"}, "secrets": [{"type": "Kubeconfig", "name": "local-cluster-kubeconfig", "key": "kubeconfig"}], }, diff --git a/e2e/clouds/test_placement.py b/e2e/clouds/test_placement.py index 0b4cfe28c..77b0af53a 100644 --- a/e2e/clouds/test_placement.py +++ b/e2e/clouds/test_placement.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The serving stack fans out to the GPU pool, and only the GPU pool. - -Two-sided checks. The fan-out test catches tolerations too narrow to reach -the tainted GPU pool, such as an Equal toleration whose value doesn't match -the taint. The audits catch the opposite class: pods tolerating taints they -shouldn't. Real AKS rejects wildcard tolerations at admission and every cloud -taints its GPU pools, so a system-pool pod carrying a wildcard or -nvidia.com/gpu toleration is a bug even when the scheduler happened to keep -it on the system node. +"""Each device stack fans out to its own pool, and only its own pool. + +Two-sided checks, per pool. The fan-out test catches tolerations too narrow +to reach a tainted pool, such as an Equal toleration whose value doesn't +match the taint, and a filter that wrongly dropped a claimed stack. The +audits catch the opposite class: pods tolerating taints they shouldn't, and +one stack's operands landing on another stack's pool - the NVIDIA kubelet +plugin on a CPU node, or dra-driver-cpu on a GPU node. Real AKS rejects +wildcard tolerations at admission and every cloud taints its inference +pools, so a system-pool pod carrying a wildcard or pool toleration is a bug +even when the scheduler happened to keep it on the system node. """ import re @@ -29,10 +31,10 @@ from e2e import clouds, kube, wait -# Pods expected to carry the GPU toleration and land on the fake GPU nodes: -# the GPU-reaching DaemonSets from generate.py's TOLERATIONS table - the NFD -# worker, node-exporter, DRA kubelet plugin, gpu-operator operands and -# nvsentinel's node monitors (platform-connectors runs on every node; the +# Pods expected to carry the NVIDIA GPU toleration and land on the fake GPU +# nodes: the GPU-reaching DaemonSets from generate.py's TOLERATIONS table - +# the NFD worker, node-exporter, DRA kubelet plugin, gpu-operator operands +# and nvsentinel's node monitors (platform-connectors runs on every node; the # health monitors select on labels nvsentinel's labeler applies at runtime). # Unanchored where the generated and hand-written halves name the same # workload differently. Per-cloud extras come from Cloud.gpu_workloads. @@ -51,9 +53,31 @@ "^platform-connectors", ) -# GPU-only pods must never run on the system pool: they'd only land there by -# a selector or affinity regression. The NFD worker and node-exporter aren't -# here - they legitimately run on every node. +# Pods expected to carry the CPU pool toleration and land on the fake CPU +# nodes: dra-driver-cpu (its pin's nodeSelector and toleration name exactly +# this pool) and the node exporter, which reaches every inference pool. +# NFD is deliberately absent: it exists to label GPU nodes, so it carries +# the GPU drivers' tags and stays off CPU pools. +CPU_WORKLOADS = ( + "dra-driver-cpu", + "prometheus-node-exporter", +) + +# Pods expected to carry the AMD GPU toleration and land on the fake AMD +# nodes (the Existing leg only): the AMD operator's operands, named for its +# chart's default DeviceConfig, plus the NFD worker - it labels both GPU +# vendors' nodes - and the node exporter. +AMD_WORKLOADS = ( + "node-feature-discovery-worker", + "prometheus-node-exporter", + "^default-dra-driver", + "^default-node-labeller", + "^default-metrics-exporter", +) + +# Pool-only pods must never run anywhere but their own pool: they'd only +# land elsewhere by a selector or affinity regression. The NFD worker and +# node-exporter aren't here - they legitimately run on several pools. GPU_ONLY = re.compile( "|".join( ( @@ -68,16 +92,45 @@ ) ) ) +CPU_ONLY = re.compile("dra-driver-cpu") +AMD_ONLY = re.compile("|".join(("^default-dra-driver", "^default-node-labeller", "^default-metrics-exporter"))) + +# The taint each fake pool carries, keyed by the pool the tests report. +POOL_TAINT_KEYS = { + "nvidia": "nvidia.com/gpu", + "cpu": "modelplane.ai/cpu", + "amd": "amd.com/gpu", +} # Cluster infrastructure the serving stack doesn't manage. EXEMPT_NAMESPACES = ("kube-system", "metallb-system", "local-path-storage") def gpu_workloads(cloud: clouds.Cloud) -> re.Pattern[str]: - """Return the pod names allowed to tolerate the GPU taint and run on the fake nodes, for a cloud.""" + """Return the pod names allowed to tolerate the GPU taint and run on the fake GPU nodes, for a cloud.""" return re.compile("|".join(GPU_WORKLOADS + cloud.gpu_workloads)) +def allowed_by_pool(cloud: clouds.Cloud) -> dict[str, re.Pattern[str]]: + """Return each pool's allowed pod names, for a cloud.""" + return { + "nvidia": gpu_workloads(cloud), + "cpu": re.compile("|".join(CPU_WORKLOADS)), + "amd": re.compile("|".join(AMD_WORKLOADS)), + } + + +def fake_pools(workload: kube.Cluster) -> dict[str, str]: + """Return each fake node's pool, from the taint its leg shaped it with.""" + taint_pools = {taint: pool for pool, taint in POOL_TAINT_KEYS.items()} + pools = {} + for node in workload.core.list_node(label_selector="type=kwok", _request_timeout=kube.TIMEOUT_SECONDS).items: + for taint in node.spec.taints or []: + if taint.key in taint_pools: + pools[node.metadata.name] = taint_pools[taint.key] + return pools + + def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: """Parametrize the fan-out test from the leg's expected DaemonSets, so a failure names its DaemonSet.""" if "daemonset" in metafunc.fixturenames: @@ -87,12 +140,13 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: @pytest.mark.usefixtures("installed") def test_daemonsets_fan_out(workload: kube.Cluster, daemonset: str, want: int) -> None: - """A DaemonSet the cloud expects on its GPU pool reaches its ready count. + """A DaemonSet the cloud expects on its device pools reaches its ready count. Charts without dependents get no helm --wait, so a rollout can lag the ServingStack's Ready condition; poll. KWOK fakes the pods on the fake nodes Running, so a shortfall means the scheduler never placed them - a - toleration too narrow to reach the tainted pool. + toleration too narrow to reach the tainted pool, a selector that misses + its nodes, or a filter that dropped the stack the pool needs. """ namespace, name = daemonset.split("/", 1) @@ -111,13 +165,13 @@ def fanned_out() -> None: @pytest.mark.usefixtures("installed") def test_tolerations_are_scoped(workload: kube.Cluster, cloud: clouds.Cloud) -> None: - """No pod outside the GPU allowlist tolerates the GPU taint, or every taint. + """No pod outside a pool's allowlist tolerates that pool's taint, or every taint. Wherever the scheduler put the pod: a wildcard that happened to stay on the system node still squats on accelerated capacity the moment the real - cloud's autoscaler adds a GPU node. + cloud's autoscaler adds a tainted node. """ - allowed = gpu_workloads(cloud) + allowed = allowed_by_pool(cloud) bad = [] for pod in workload.core.list_pod_for_all_namespaces(_request_timeout=kube.TIMEOUT_SECONDS).items: namespace, name = pod.metadata.namespace, pod.metadata.name @@ -126,27 +180,36 @@ def test_tolerations_are_scoped(workload: kube.Cluster, cloud: clouds.Cloud) -> for toleration in pod.spec.tolerations or []: if toleration.operator == "Exists" and not toleration.key: bad.append(f"{namespace}/{name} tolerates every taint") - elif toleration.key == "nvidia.com/gpu" and not allowed.search(name): - bad.append(f"{namespace}/{name} tolerates the GPU taint") + continue + for pool, taint_key in POOL_TAINT_KEYS.items(): + if toleration.key == taint_key and not allowed[pool].search(name): + bad.append(f"{namespace}/{name} tolerates the {pool} pool's taint") assert bad == [], "pods carrying tolerations the system pool must not have" @pytest.mark.usefixtures("installed") -def test_only_gpu_workloads_on_the_gpu_pool(workload: kube.Cluster, cloud: clouds.Cloud) -> None: - """Everything on a fake GPU node is an expected GPU workload, and GPU-only workloads stay off the system pool.""" - allowed = gpu_workloads(cloud) - fake = { - node.metadata.name - for node in workload.core.list_node(label_selector="type=kwok", _request_timeout=kube.TIMEOUT_SECONDS).items - } - assert fake, "the workload cluster has no fake GPU nodes" +def test_pools_hold_their_own(workload: kube.Cluster, cloud: clouds.Cloud) -> None: + """Everything on a fake node belongs to that node's pool, and pool-only workloads stay there. + + The second side is what gives the filter teeth: dra-driver-cpu off the + CPU pool, or a GPU stack's operand off its GPU pool, means a selector or + toleration regressed - or a device stack installed somewhere its driver + was never claimed. + """ + allowed = allowed_by_pool(cloud) + pools = fake_pools(workload) + assert pools, "the workload cluster has no fake nodes" + only = {"nvidia": GPU_ONLY, "cpu": CPU_ONLY, "amd": AMD_ONLY} bad = [] for pod in workload.core.list_pod_for_all_namespaces(_request_timeout=kube.TIMEOUT_SECONDS).items: namespace, name, node = pod.metadata.namespace, pod.metadata.name, pod.spec.node_name if namespace in EXEMPT_NAMESPACES or node is None: continue - if node in fake and not allowed.search(name): - bad.append(f"{namespace}/{name} doesn't belong on fake GPU node {node}") - elif node not in fake and GPU_ONLY.search(name): - bad.append(f"{namespace}/{name} is GPU-only but sits on system node {node}") + pool = pools.get(node) + if pool is not None and not allowed[pool].search(name): + bad.append(f"{namespace}/{name} doesn't belong on fake {pool} node {node}") + for pool_only, pattern in only.items(): + if pool != pool_only and pattern.search(name): + where = f"fake {pool} node {node}" if pool else f"system node {node}" + bad.append(f"{namespace}/{name} is {pool_only}-only but sits on {where}") assert bad == [], "pods scheduled onto the wrong pool" diff --git a/e2e/environment.py b/e2e/environment.py index 69ce0de7a..a792522cf 100644 --- a/e2e/environment.py +++ b/e2e/environment.py @@ -94,9 +94,20 @@ def up(*, apply_manifests: bool, cloud: clouds.Cloud | None = None) -> None: log.info("Applying %s", MANIFESTS) kubectl(CONTROL_PLANE_CONTEXT, "apply", f"--filename={MANIFESTS}") return - log.info("Applying the fake GPU nodes and the %s ServingStack", cloud.cloud) + log.info("Applying the fake device-pool nodes and the %s ServingStack", cloud.cloud) nodes = {"apiVersion": "v1", "kind": "List", "items": clouds.nodes(cloud)} kubectl(WORKLOAD_CONTEXT, "apply", "--filename=-", stdin=json.dumps(nodes)) + # Apply resets the fake nodes this leg shares with a previous one, but + # never prunes whole objects: a leg with fewer pools reusing a cluster + # (any leg after Existing's AMD pool) would keep stale fake nodes whose + # taints the placement audits then reason about. Delete the leftovers. + wanted = {node["metadata"]["name"] for node in clouds.nodes(cloud)} + observed = output( + "kubectl", f"--context={WORKLOAD_CONTEXT}", + "get", "nodes", "--selector=type=kwok", "--output=json", + ) # fmt: skip + for stale in sorted({node["metadata"]["name"] for node in json.loads(observed)["items"]} - wanted): + kubectl(WORKLOAD_CONTEXT, "delete", "node", stale) kubectl(CONTROL_PLANE_CONTEXT, "apply", "--filename=-", stdin=json.dumps(clouds.serving_stack(cloud))) diff --git a/e2e/test_legs.py b/e2e/test_legs.py index 3e4b07d53..f28e80f14 100644 --- a/e2e/test_legs.py +++ b/e2e/test_legs.py @@ -80,3 +80,27 @@ def test_taint_matches_composition(leg: str) -> None: f"the {leg} leg fakes nvidia.com/gpu={clouds.CLOUDS[leg].taint_value}, " f"but {COMPOSITIONS[leg]} taints GPU pools with {value}" ) + + +# The compositions whose CPU pool label-and-taint the legs' fake CPU nodes +# copy. GKE is included here, unlike COMPOSITIONS: its GPU taint comes from +# GKE itself, but compose-gke-cluster sets the CPU taint explicitly, since +# GKE only auto-taints GPU pools. Still no Existing entry: Modelplane +# doesn't taint BYO pools. +CPU_COMPOSITIONS = {**COMPOSITIONS, "gke": "compose-gke-cluster"} + + +@pytest.mark.parametrize("leg", sorted(CPU_COMPOSITIONS)) +def test_cpu_taint_matches(leg: str) -> None: + """The fake CPU pools' taint key is the one every cluster composition applies. + + The compositions share one key (_LABEL_CPU doubles as the taint key and + the label dra-driver-cpu's nodeSelector reads), so the legs share one + CPU_TAINT rather than faking it per cloud - this pins both halves of + that assumption. + """ + consts = constants(CPU_COMPOSITIONS[leg]) + assert consts["_LABEL_CPU"] == clouds.CPU_TAINT["key"], ( + f"the legs fake the CPU taint {clouds.CPU_TAINT['key']}, " + f"but {CPU_COMPOSITIONS[leg]} labels and taints CPU pools with {consts['_LABEL_CPU']}" + ) diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index 4e6ee7b99..bb3f02ee4 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -2328,7 +2328,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, @@ -2380,7 +2390,17 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule", - } + }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + { + "key": "modelplane.ai/cpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, diff --git a/functions/compose-serving-stack/function/stacks/common.py b/functions/compose-serving-stack/function/stacks/common.py index 7296b25df..037f3c410 100644 --- a/functions/compose-serving-stack/function/stacks/common.py +++ b/functions/compose-serving-stack/function/stacks/common.py @@ -320,13 +320,14 @@ def _crds(filename: str) -> list[dict[str, Any]]: # engine container to a cpuset. Pinned here because a CPU node needs # the same driver whatever provisioned it, so every cloud - Existing # included - installs the one pin. The nodeSelector keeps it to CPU - # pools, which every cluster composition labels modelplane.ai/cpu: - # the chart's default tolerations accept every NoSchedule taint, so - # the selector is what keeps it off GPU and system pools, not their - # taints. reservedCPUs holds cores 0-1 back for the OS and kubelet, - # so a class's capacity.cpu is the node's vCPUs less two - a fixed, - # documented value rather than a per-pool knob someone must keep in - # step with the class by hand. + # pools, which every cluster composition labels modelplane.ai/cpu; + # the tolerations are set because the chart's default is a keyless + # wildcard, which AKS rejects at admission and the e2e toleration + # audit flags - the CPU taint is the only one the selector can land + # the driver behind. reservedCPUs holds cores 0-1 back for the OS + # and kubelet, so a class's capacity.cpu is the node's vCPUs less + # two - a fixed, documented value rather than a per-pool knob + # someone must keep in step with the class by hand. Chart( key="dra-driver-cpu", release="mp-dra-driver-cpu", @@ -337,6 +338,9 @@ def _crds(filename: str) -> list[dict[str, Any]]: dra_drivers=["dra.cpu"], values={ "nodeSelector": {"modelplane.ai/cpu": "true"}, + "tolerations": [ + {"key": "modelplane.ai/cpu", "operator": "Exists", "effect": "NoSchedule"}, + ], "driverConfig": {"reservedCPUs": "0-1"}, }, ), From 9b411845de789049dc298069240dc0006a0eba26 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 22 Oct 2026 16:43:42 +0200 Subject: [PATCH 3/4] feat(recipes): add recipes, update telemetry for amd Signed-off-by: Christopher Haar --- .../config/vocabularies/Modelplane/accept.txt | 18 +++++++ apis/metricmappings/definition.yaml | 12 ++--- docs/content/platform/telemetry.md | 32 +++++++++--- docs/content/recipes/qwen2.5-0.5b-cpu.md | 50 +++++++++++++++++++ docs/content/recipes/qwen2.5-0.5b.md | 49 ++++++++++++++++++ docs/data/recipes.yaml | 17 ++++++- .../qwen2.5-0.5b-cpu/inference-class.yaml | 29 +++++++++++ .../qwen2.5-0.5b-cpu/inference-cluster.yaml | 19 +++++++ .../qwen2.5-0.5b-cpu/model-deployment.yaml | 42 ++++++++++++++++ .../qwen2.5-0.5b-cpu/model-service.yaml | 15 ++++++ .../recipes/qwen2.5-0.5b/inference-class.yaml | 29 +++++++++++ .../qwen2.5-0.5b/inference-cluster.yaml | 20 ++++++++ .../qwen2.5-0.5b/model-deployment.yaml | 36 +++++++++++++ .../recipes/qwen2.5-0.5b/model-service.yaml | 15 ++++++ .../function/collector.py | 34 ++++++++----- .../function/stacks/metrics.py | 27 ++++++++++ .../tests/test_collector.py | 38 +++++++++++--- .../compose-serving-stack/tests/test_fn.py | 28 ++++++++--- .../tests/test_metrics.py | 6 ++- schemas/.lock.json | 2 +- .../ai/modelplane/metricmapping/v1alpha1.py | 11 +++- 21 files changed, 484 insertions(+), 45 deletions(-) create mode 100644 docs/content/recipes/qwen2.5-0.5b-cpu.md create mode 100644 docs/content/recipes/qwen2.5-0.5b.md create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml create mode 100644 docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml diff --git a/.vale/styles/config/vocabularies/Modelplane/accept.txt b/.vale/styles/config/vocabularies/Modelplane/accept.txt index 6ea9f82b9..b48c1c65b 100644 --- a/.vale/styles/config/vocabularies/Modelplane/accept.txt +++ b/.vale/styles/config/vocabularies/Modelplane/accept.txt @@ -115,6 +115,15 @@ vLLM [Hh]uggingFace NVIDIA NVLink +AMD +Instinct +ROCm +amdgpu +CUDA +HBM3e +cpuset +device-metrics-exporter +[Mm]icrojoules Envoy Traefik MetalLB @@ -725,6 +734,15 @@ vLLM [Hh]uggingFace NVIDIA NVLink +AMD +Instinct +ROCm +amdgpu +CUDA +HBM3e +cpuset +device-metrics-exporter +[Mm]icrojoules Envoy Traefik MetalLB diff --git a/apis/metricmappings/definition.yaml b/apis/metricmappings/definition.yaml index 63b512801..31a8a99fd 100644 --- a/apis/metricmappings/definition.yaml +++ b/apis/metricmappings/definition.yaml @@ -150,16 +150,16 @@ spec: as the component wrote it. fromUnit: type: string - enum: [Millijoules, Mebibytes, Milliseconds, Nanoseconds, Percent] + enum: [Millijoules, Microjoules, Mebibytes, Milliseconds, Nanoseconds, Percent] description: >- What the component measures this in, when that isn't the unit the name claims. Modelplane converts to the base unit: millijoules and milliseconds are divided by - a thousand, nanoseconds by a billion, percent by a - hundred, and mebibytes multiplied out to bytes. A - histogram is converted whole - its sum, its bounds and - its bucket boundaries - so its quantiles come out in - the target unit too. + a thousand, microjoules by a million, nanoseconds by a + billion, percent by a hundred, and mebibytes multiplied + out to bytes. A histogram is converted whole - its sum, + its bounds and its bucket boundaries - so its quantiles + come out in the target unit too. Percent is for a component that counts a saturation from nought to a hundred where the name says a ratio. diff --git a/docs/content/platform/telemetry.md b/docs/content/platform/telemetry.md index 99f565e15..b63fc8c54 100644 --- a/docs/content/platform/telemetry.md +++ b/docs/content/platform/telemetry.md @@ -93,7 +93,7 @@ Some examples of the available metrics: Latency appears twice on purpose. The `frontend_` series are what your caller experienced, measured at the gateway. The engine's own series are what the engine spent. For -example, if the frontend metric is slow and the engine isn't, you can +example, if the frontend metric is slow and the engine isn't, you can troubleshoot routing, queueing, or networking issues instead of the model. Saturation gauges are per replica, so how you combine them decides what you see. Average @@ -223,7 +223,7 @@ Anything else the exporter takes goes under `config`, passed through as you wrot Modelplane doesn't define a schema for an exporter's settings so anything under the `config` is passed to the collector exactly as written. TLS, retries, queueing, compression and headers all work and any new settings in the collector -are respected and the sink keeps working. +are respected and the sink keeps working. To use an authentication scheme Modelplane doesn't compose, define the extension yourself under `spec.extensions`. Then reference it by its key from the sink's @@ -459,12 +459,16 @@ spec: ## GPUs -The GPU series come from a [DCGM -exporter](https://github.com/NVIDIA/dcgm-exporter), not from the engine: a GPU +The GPU series come from a vendor's GPU exporter, not from the engine: a GPU belongs to a node, and its series have a `node` label rather than a -deployment. The collector discovers the exporter by its pod - an `app` or -`app.kubernetes.io/name` label containing `dcgm`, with a container port named -`metrics` - and scrapes it directly, not through Prometheus. +deployment. On NVIDIA pools that's a [DCGM +exporter](https://github.com/NVIDIA/dcgm-exporter); on AMD pools it's the AMD +GPU operator's +[device-metrics-exporter](https://github.com/ROCm/device-metrics-exporter), +which the serving stack installs with the AMD stack. The collector discovers +either by its pod - an `app` or `app.kubernetes.io/name` label containing +`dcgm` or `metrics-exporter`, with a container port named `metrics` or +`exporter-port` - and scrapes it directly, not through Prometheus. Where the exporter comes from depends on the cluster: @@ -504,6 +508,20 @@ curl -s localhost:9400/metrics | grep -c -E \ 'DCGM_FI_DEV_FB_(USED|FREE)|DCGM_FI_PROF_(GR_ENGINE_ACTIVE|PIPE_TENSOR_ACTIVE|DRAM_ACTIVE)|DCGM_FI_DEV_(GPU_TEMP|POWER_USAGE|TOTAL_ENERGY_CONSUMPTION)' ``` +On an AMD pool the same `modelplane_*` names come from the +device-metrics-exporter's fields instead. A dashboard built on them needs no +change when a deployment moves between vendors' pools. + +| AMD field | Becomes | Unit | +| --- | --- | --- | +| `gpu_used_vram` | `modelplane_gpu_memory_used_bytes` | MiB, converted to bytes | +| `gpu_free_vram` | `modelplane_gpu_memory_free_bytes` | MiB, converted to bytes | +| `gpu_gfx_activity` | `modelplane_gpu_compute_active_ratio` | percent, converted to a ratio | +| `gpu_umc_activity` | `modelplane_gpu_memory_bandwidth_ratio` | percent, converted to a ratio | +| `gpu_junction_temperature` | `modelplane_gpu_temperature_celsius` | Celsius | +| `gpu_power_usage` | `modelplane_gpu_power_watts` | watts | +| `gpu_energy_consumed` | `modelplane_energy_joules_total` | microjoules, converted to joules | + If fields are missing, extend the exporter's metrics ConfigMap - on a GPU Operator install that's the `dcgmExporter.config` chart value. The configuration Modelplane installs on the clouds it provisions publishes each diff --git a/docs/content/recipes/qwen2.5-0.5b-cpu.md b/docs/content/recipes/qwen2.5-0.5b-cpu.md new file mode 100644 index 000000000..f37ffba56 --- /dev/null +++ b/docs/content/recipes/qwen2.5-0.5b-cpu.md @@ -0,0 +1,50 @@ +--- +title: Qwen2.5-0.5B on CPU +weight: 14 +description: A 0.5B dense chat model served on CPUs alone, on a Civo CPU pool. +model: Qwen/Qwen2.5-0.5B-Instruct +vendors: [Qwen] +clouds: [Civo] +accelerators: [CPU] +engines: [vLLM] +arch: Dense +precisions: ["BF16"] +size: 0.5B +ctx: "4,096" +servingModes: [Standalone] +engineImages: [vllm/vllm-openai-cpu:v0.32.0-x86_64] +gpuNote: no GPU +--- + +A 0.5B dense chat model served on CPUs alone, on a Civo `g4p.kube.medium` +pool: one `Standalone` engine, no cache, weights pulled straight from Hugging +Face. A CPU pool is the cheapest way to exercise a Modelplane deployment end +to end - no GPU to find, no driver to wait for - and a 0.5B model is small +enough that CPU inference answers in sensible time. + +The class's provisioning block names no accelerator, which is what makes the +pool a CPU pool: the cluster composition builds it with a standard node image +and the `modelplane.ai/cpu` label and taint, and the serving stack installs +dra-driver-cpu - and no GPU machinery - because the class claims `dra.cpu`. +The claim pins the engine to its NUMA node's cpuset. The engine is vLLM's CPU +build, whose image serves the same OpenAI API as the CUDA one. The same class +without its provisioning block describes a bring-your-own CPU pool, on any +conformant cluster: label its nodes `modelplane.ai/pool=cpu-pool` and +`modelplane.ai/cpu=true` instead. + +## Validated deployments + +{{< validated-deployments >}} + +## Platform + +{{< manifests "recipes/qwen2.5-0.5b-cpu/inference-class.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml" >}} + +## Deployment + +{{< manifests "recipes/qwen2.5-0.5b-cpu/model-deployment.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b-cpu/model-service.yaml" >}} + diff --git a/docs/content/recipes/qwen2.5-0.5b.md b/docs/content/recipes/qwen2.5-0.5b.md new file mode 100644 index 000000000..611a40a3f --- /dev/null +++ b/docs/content/recipes/qwen2.5-0.5b.md @@ -0,0 +1,49 @@ +--- +title: Qwen2.5-0.5B on AMD +weight: 13 +description: A 0.5B dense chat model on a single AMD Instinct MI325X on a bring-your-own cluster. +model: Qwen/Qwen2.5-0.5B-Instruct +vendors: [Qwen] +clouds: [Existing] +accelerators: [MI325X] +engines: [vLLM] +arch: Dense +precisions: ["BF16"] +size: 0.5B +ctx: "32,768" +servingModes: [Standalone] +engineImages: [rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0] +gpuNote: 1× of 8 per node +--- + +A 0.5B dense chat model on a single AMD Instinct MI325X on a bring-your-own +cluster: one `Standalone` engine, no cache, weights pulled straight from +Hugging Face. The model is deliberately small - the point of this run is the +AMD serving path, not the model. The class claims the GPU through DRA +(`gpu.amd.com`); the serving stack installs the AMD GPU operator in DRA mode +when a class claims that driver, against the amdgpu kernel driver the node +image already carries. The engine is AMD's ROCm vLLM. + +The manifests are the ones validated end to end on an 8x MI325X bare metal +node (each GPU has just under 256 GiB of HBM3e), re-registered here as a +bring-your-own cluster - `source: Existing` is the one cloud source that +installs the AMD stack today. Bring the cluster with the amdgpu driver on its +GPU nodes, label the pool's nodes `modelplane.ai/pool=gpu-mi325x`, and apply +the platform side first, then the ML side. + +## Validated deployments + +{{< validated-deployments >}} + +## Platform + +{{< manifests "recipes/qwen2.5-0.5b/inference-class.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b/inference-cluster.yaml" >}} + +## Deployment + +{{< manifests "recipes/qwen2.5-0.5b/model-deployment.yaml" >}} + +{{< manifests "recipes/qwen2.5-0.5b/model-service.yaml" >}} + diff --git a/docs/data/recipes.yaml b/docs/data/recipes.yaml index 12f0c4871..65a7c9f5b 100644 --- a/docs/data/recipes.yaml +++ b/docs/data/recipes.yaml @@ -39,17 +39,32 @@ clouds: - name: Vultr term: Vultr icon: brand-vultr -gpus: + - name: Civo + term: Civo + icon: brand-civo + - name: Bring your own + term: Existing +accelerators: - name: A16 mem: 16G + icon: brand-nvidia - name: L4 mem: 24G + icon: brand-nvidia - name: A100 mem: 40/80G + icon: brand-nvidia - name: H100 mem: 80G + icon: brand-nvidia - name: H200 mem: 141G + icon: brand-nvidia + - name: MI325X + mem: 256G + icon: brand-amd + - name: CPU + icon: cpu servingModes: - Standalone - LeaderWorker diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml new file mode 100644 index 000000000..7a3e5a149 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-class.yaml @@ -0,0 +1,29 @@ +# An InferenceClass describing a Civo CPU node pool - no GPU at all. +# +# The provisioning block has no accelerator, so the cluster composition +# builds the pool with the CPU role: a standard node image and the +# modelplane.ai/cpu label and taint, nothing GPU. The serving stack installs +# dra-driver-cpu when a class claims dra.cpu, which publishes each node's +# cores as claimable devices (one device per NUMA node) and registers the +# dra.cpu DeviceClass. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceClass +metadata: + name: civo-cpu-8vcpu +spec: + description: "Civo g4p.kube.medium, 8 vCPUs, no GPU" + provisioning: + provider: Civo + civo: + size: g4p.kube.medium + devices: + - name: cpu + claim: DRA + driver: dra.cpu + deviceClassName: dra.cpu + count: 1 + capacity: + # The node's 8 vCPUs less the two dra-driver-cpu reserves for the OS + # and kubelet (reservedCPUs 0-1), which is what the driver's + # ResourceSlices report as claimable. + cpu: { value: "6" } diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml new file mode 100644 index 000000000..ad66dd1f1 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/inference-cluster.yaml @@ -0,0 +1,19 @@ +# A Civo cluster in NYC1 with a single CPU node offering the class above. +# The composed ServingStack gets draDrivers: [dra.cpu], so dra-driver-cpu +# installs and the GPU device stacks stay out - a cluster with no GPU runs +# no GPU machinery. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: civo-cpu + labels: + modelplane.ai/region: nyc1 +spec: + cluster: + source: Civo + civo: + region: NYC1 + nodePools: + - name: cpu-pool + className: civo-cpu-8vcpu + nodeCount: 1 diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml new file mode 100644 index 000000000..0396b87d4 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-deployment.yaml @@ -0,0 +1,42 @@ +# A small model served on CPUs alone. The device selector matches against +# the claimable cores the cpu-8vcpu class declares, so >= 4 lands it on the +# CPU pool; the claim takes the whole NUMA-node device and pins the engine +# container to its cpuset. The engine is vLLM's CPU build - the default +# vllm/vllm-openai image is CUDA-only. --block-size=16 matches the block +# size the endpoint picker assumes when an engine doesn't state one, so +# prefix-cache routing keeps working; VLLM_CPU_KVCACHE_SPACE sets the KV +# cache's share of the node's RAM in GiB. No ModelCache: the engine pulls +# weights straight from Hugging Face. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen-cpu-demo + namespace: ml-team +spec: + replicas: 1 + template: + spec: + engines: + - name: qwen + members: + - role: Standalone + nodeSelector: + devices: + - name: cpu + count: 1 + selectors: + - cel: | + device.capacity["dra.cpu"].cpu.compareTo(quantity("4")) >= 0 + template: + spec: + containers: + - name: engine + image: vllm/vllm-openai-cpu:v0.32.0-x86_64 + args: + - --model=Qwen/Qwen2.5-0.5B-Instruct + - --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME) + - --max-model-len=4096 + - --block-size=16 + env: + - name: VLLM_CPU_KVCACHE_SPACE + value: "4" diff --git a/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml new file mode 100644 index 000000000..eea7f0fe4 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b-cpu/model-service.yaml @@ -0,0 +1,15 @@ +# Publishes the deployment's endpoints as one model a caller can name: +# /, so ml-team/qwen-cpu here. Modelplane composes one +# ModelEndpoint per ModelReplica, labeled with the deployment name, so +# this selector reaches every replica. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen-cpu + namespace: ml-team +spec: + endpoints: + - name: qwen-cpu-demo + selector: + matchLabels: + modelplane.ai/deployment: qwen-cpu-demo diff --git a/docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml b/docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml new file mode 100644 index 000000000..20eef725a --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/inference-class.yaml @@ -0,0 +1,29 @@ +# An InferenceClass describing a BYO 8x AMD Instinct MI325X node pool. +# +# No provisioning block: this class describes hardware that already exists on +# a bring-your-own cluster whose node images carry the amdgpu kernel driver. +# When a cluster references it, the serving stack installs the AMD GPU +# operator in DRA mode (driver management off), whose DRA driver publishes +# the GPUs as gpu.amd.com ResourceSlices. Label the pool's nodes +# modelplane.ai/pool=; the amd.com/gpu=true:NoSchedule taint is +# the convention that keeps system workloads off them, and engine pods +# tolerate it. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceClass +metadata: + name: amd-mi325x-8x +spec: + description: "BYO 8x AMD Instinct MI325X 256GB" + devices: + - name: gpu + claim: DRA + driver: gpu.amd.com + deviceClassName: gpu.amd.com + count: 8 + attributes: + productName: { string: AMD_Instinct_MI325_OAM } + capacity: + # The MI325X's usable HBM3e as the AMD DRA driver's ResourceSlices + # report it, not its nominal 256GB. A nodeSelector asking for + # >= 256Gi would never bind. + memory: { value: "262128Mi" } diff --git a/docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml b/docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml new file mode 100644 index 000000000..7523e172e --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/inference-cluster.yaml @@ -0,0 +1,20 @@ +# Registers the bring-your-own cluster and installs the serving stack onto +# it through the kubeconfig Secret. The composed ServingStack gets +# draDrivers: [gpu.amd.com], so the AMD GPU operator (DRA mode) installs and +# the NVIDIA and CPU device stacks stay out. Existing is the one cloud that +# installs the AMD stack today. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: amd-byo +spec: + cluster: + source: Existing + existing: + secretRef: + name: byo-cluster-kubeconfig + key: kubeconfig + nodePools: + - name: gpu-mi325x + className: amd-mi325x-8x + nodeCount: 1 diff --git a/docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml b/docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml new file mode 100644 index 000000000..f22ef3fb6 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/model-deployment.yaml @@ -0,0 +1,36 @@ +# A small model to prove placement and serving on the AMD pool. The device +# selector matches against the capacity the amd-mi325x-8x class declares +# (262128Mi), not the pod's resource requests, so >= 200Gi lands it on the +# MI325X pool; the claim takes 1 of the node's 8 GPUs. The engine is AMD's +# ROCm vLLM build (the CDNA variant covers MI325X/gfx942) - the upstream +# vllm/vllm-openai image is CUDA-only. Unlike that image, rocm/vllm's +# entrypoint is not the OpenAI server, so the command invokes it explicitly. +# No ModelCache: the engine pulls weights straight from Hugging Face. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen-demo + namespace: ml-team +spec: + replicas: 1 + template: + spec: + engines: + - name: qwen + members: + - role: Standalone + nodeSelector: + devices: + - name: gpu + count: 1 + selectors: + - cel: | + device.capacity["gpu.amd.com"].memory.compareTo(quantity("200Gi")) >= 0 + template: + spec: + containers: + - name: engine + image: rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0 + command: ["vllm", "serve", "Qwen/Qwen2.5-0.5B-Instruct"] + args: + - --served-model-name=$(MODELPLANE_SERVED_MODEL_NAME) diff --git a/docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml b/docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml new file mode 100644 index 000000000..41341ec24 --- /dev/null +++ b/docs/manifests/recipes/qwen2.5-0.5b/model-service.yaml @@ -0,0 +1,15 @@ +# Publishes the deployment's endpoints as one model a caller can name: +# /, so ml-team/qwen here. Modelplane composes one +# ModelEndpoint per ModelReplica, labeled with the deployment name, so +# this selector reaches every replica. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen + namespace: ml-team +spec: + endpoints: + - name: qwen-demo + selector: + matchLabels: + modelplane.ai/deployment: qwen-demo diff --git a/functions/compose-serving-stack/function/collector.py b/functions/compose-serving-stack/function/collector.py index 8d73b6543..a3814179c 100644 --- a/functions/compose-serving-stack/function/collector.py +++ b/functions/compose-serving-stack/function/collector.py @@ -139,14 +139,17 @@ def _relabel_pod_identity() -> list[dict[str, Any]]: ] + [{"source_labels": ["__meta_kubernetes_namespace"], "target_label": "namespace"}] -def _dcgm_selector(action: str) -> dict[str, Any]: - """Match a GPU exporter, however it was packaged. - - The two spell it differently - gke-managed-dcgm-exporter and dcgm-exporter - - and put the name on different labels depending on who packaged it, so both - are read and the match is on what they share. One definition, used by the - job that keeps them and the job that has to leave them alone, because the - two drifting apart is how a series gets collected twice. +def _gpu_exporter_selector(action: str) -> dict[str, Any]: + """Match a GPU exporter, whichever vendor's and however it was packaged. + + NVIDIA's DCGM exporters spell it differently - gke-managed-dcgm-exporter + and dcgm-exporter - and put the name on different labels depending on who + packaged it, so both labels are read and the match is on what they share. + The AMD GPU operator names its device-metrics-exporter pods + app.kubernetes.io/name: metrics-exporter, which the same pattern's second + alternative matches. One definition, used by the job that keeps them and + the job that has to leave them alone, because the two drifting apart is + how a series gets collected twice. """ return { "source_labels": [ @@ -154,7 +157,7 @@ def _dcgm_selector(action: str) -> dict[str, Any]: "__meta_kubernetes_pod_label_app", ], "action": action, - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", } @@ -340,8 +343,14 @@ def _scrape_configs() -> list[dict[str, Any]]: # two spell it differently: gke-managed-dcgm-exporter and # dcgm-exporter. Both labels are read, since which one carries # the name depends on who packaged it. - _dcgm_selector("keep"), - {"source_labels": ["__meta_kubernetes_pod_container_port_name"], "action": "keep", "regex": "metrics"}, + _gpu_exporter_selector("keep"), + # The exporters name their metrics port differently: DCGM's + # is metrics, the AMD operator's is exporter-port. + { + "source_labels": ["__meta_kubernetes_pod_container_port_name"], + "action": "keep", + "regex": "metrics|exporter-port", + }, # The node, because a GPU series belongs to hardware rather # than to a deployment. DCGM names the card itself. {"source_labels": ["__meta_kubernetes_pod_node_name"], "target_label": "node"}, @@ -371,7 +380,7 @@ def _scrape_configs() -> list[dict[str, Any]]: "action": "drop", "regex": ".+", }, - _dcgm_selector("drop"), + _gpu_exporter_selector("drop"), # The rest of the same convention, not just the first line of # it. A pod that says scrape me generally also says where: the # cert-manager webhook declares 10250 first and annotates 9402, @@ -411,6 +420,7 @@ def _scrape_configs() -> list[dict[str, Any]]: # literal in that position. _UNIT_FACTOR = { "Millijoules": "0.001", + "Microjoules": "0.000001", "Milliseconds": "0.001", "Nanoseconds": "0.000000001", "Mebibytes": "1048576.0", diff --git a/functions/compose-serving-stack/function/stacks/metrics.py b/functions/compose-serving-stack/function/stacks/metrics.py index 5d534819c..ebf12b756 100644 --- a/functions/compose-serving-stack/function/stacks/metrics.py +++ b/functions/compose-serving-stack/function/stacks/metrics.py @@ -139,6 +139,32 @@ "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION": "Millijoules", } +# The AMD GPUs, through the AMD GPU operator's device-metrics-exporter, onto +# the same modelplane_gpu_* names the DCGM series map to, so a dashboard reads +# one surface whichever vendor's pool served it. The exporter reports VRAM in +# MiB, activity in percent and energy in microjoules, each renamed onto a name +# that claims a different unit, so each says so. Junction temperature is the +# one the Instinct MI3xx series reports (MI2xx reports edge instead); no AMD +# series measures what DCGM's tensor-pipe activity does, so +# modelplane_gpu_tensor_active_ratio stays NVIDIA-only. +_AMD_GPU = { + "gpu_used_vram": "modelplane_gpu_memory_used_bytes", + "gpu_free_vram": "modelplane_gpu_memory_free_bytes", + "gpu_gfx_activity": "modelplane_gpu_compute_active_ratio", + "gpu_umc_activity": "modelplane_gpu_memory_bandwidth_ratio", + "gpu_junction_temperature": "modelplane_gpu_temperature_celsius", + "gpu_power_usage": "modelplane_gpu_power_watts", + "gpu_energy_consumed": "modelplane_energy_joules_total", +} + +_AMD_GPU_UNITS = { + "gpu_used_vram": "Mebibytes", + "gpu_free_vram": "Mebibytes", + "gpu_gfx_activity": "Percent", + "gpu_umc_activity": "Percent", + "gpu_energy_consumed": "Microjoules", +} + # How each request vLLM finished ended, by vLLM's finished_reason renamed to # reason: stop, length and abort at v0.9.2, and error and repetition as well @@ -227,6 +253,7 @@ def mappings() -> list[v1alpha1.MetricMapping]: _mapping("modelplane-sglang", _SGLANG, tokens=_SGLANG_TOKENS, extra=(_SGLANG_RESPONSES,)), _mapping("modelplane-picker", _PICKER), _mapping("modelplane-gpu", _GPU, _GPU_UNITS), + _mapping("modelplane-gpu-amd", _AMD_GPU, _AMD_GPU_UNITS), ] diff --git a/functions/compose-serving-stack/tests/test_collector.py b/functions/compose-serving-stack/tests/test_collector.py index 183192334..2a7910fd1 100644 --- a/functions/compose-serving-stack/tests/test_collector.py +++ b/functions/compose-serving-stack/tests/test_collector.py @@ -319,12 +319,12 @@ def _config( "__meta_kubernetes_pod_label_app", ], "action": "keep", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_container_port_name"], "action": "keep", - "regex": "metrics", + "regex": "metrics|exporter-port", }, {"source_labels": ["__meta_kubernetes_pod_node_name"], "target_label": "node"}, ], @@ -361,7 +361,7 @@ def _config( "__meta_kubernetes_pod_label_app", ], "action": "drop", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_annotation_prometheus_io_path"], @@ -446,6 +446,11 @@ def _config( 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_USED"', 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_FREE"', 'scale_metric(0.001) where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'scale_metric(1048576.0) where metric.name == "gpu_used_vram"', + 'scale_metric(1048576.0) where metric.name == "gpu_free_vram"', + 'scale_metric(0.01) where metric.name == "gpu_gfx_activity"', + 'scale_metric(0.01) where metric.name == "gpu_umc_activity"', + 'scale_metric(0.000001) where metric.name == "gpu_energy_consumed"', ], }, { @@ -511,6 +516,13 @@ def _config( 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "DCGM_FI_DEV_GPU_TEMP"', 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "DCGM_FI_DEV_POWER_USAGE"', 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "gpu_used_vram"', + 'set(metric.name, "modelplane_gpu_memory_free_bytes") where metric.name == "gpu_free_vram"', + 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "gpu_gfx_activity"', + 'set(metric.name, "modelplane_gpu_memory_bandwidth_ratio") where metric.name == "gpu_umc_activity"', + 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "gpu_junction_temperature"', + 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "gpu_power_usage"', + 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "gpu_energy_consumed"', ], }, ] @@ -916,7 +928,7 @@ def test_in_flight(case: InFlightCase) -> None: # because what the collector compiles them to is what this case pins. StatementsCase( name="BuiltIn", - reason="The built-in mappings convert DCGM's units, then rename every series they map.", + reason="The built-in mappings convert the GPU exporters' units, then rename every series they map.", mappings=list(stacks.BUILTIN_MAPPINGS), want=( [ @@ -931,6 +943,11 @@ def test_in_flight(case: InFlightCase) -> None: 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_USED"', 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_FREE"', 'scale_metric(0.001) where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'scale_metric(1048576.0) where metric.name == "gpu_used_vram"', + 'scale_metric(1048576.0) where metric.name == "gpu_free_vram"', + 'scale_metric(0.01) where metric.name == "gpu_gfx_activity"', + 'scale_metric(0.01) where metric.name == "gpu_umc_activity"', + 'scale_metric(0.000001) where metric.name == "gpu_energy_consumed"', ], [ 'set(datapoint.attributes["direction"], "input") where metric.name == "vllm:request_prompt_tokens_sum"', @@ -993,6 +1010,13 @@ def test_in_flight(case: InFlightCase) -> None: 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "DCGM_FI_DEV_GPU_TEMP"', 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "DCGM_FI_DEV_POWER_USAGE"', 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "gpu_used_vram"', + 'set(metric.name, "modelplane_gpu_memory_free_bytes") where metric.name == "gpu_free_vram"', + 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "gpu_gfx_activity"', + 'set(metric.name, "modelplane_gpu_memory_bandwidth_ratio") where metric.name == "gpu_umc_activity"', + 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "gpu_junction_temperature"', + 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "gpu_power_usage"', + 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "gpu_energy_consumed"', ], ), ), @@ -1193,7 +1217,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="4811f4b31c39a4f1", + config_hash="ea4cd62b47599725", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -1238,7 +1262,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="978ab36bf03ec977", + config_hash="8ff4aaf5ef60c109", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -1305,7 +1329,7 @@ def test_transform(case: TransformCase) -> None: ( "collector", _deployment( - config_hash="de542e1c484b7ed1", + config_hash="4de3e747736d84b2", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-vendor", "secret": {"secretName": "vendor-token"}}, diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index 7ad56fb7d..650ddf7ed 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -1529,12 +1529,12 @@ def _collector_config( "__meta_kubernetes_pod_label_app", ], "action": "keep", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_container_port_name"], "action": "keep", - "regex": "metrics", + "regex": "metrics|exporter-port", }, {"source_labels": ["__meta_kubernetes_pod_node_name"], "target_label": "node"}, ], @@ -1567,7 +1567,7 @@ def _collector_config( "__meta_kubernetes_pod_label_app", ], "action": "drop", - "regex": ".*dcgm.*", + "regex": ".*dcgm.*|.*metrics-exporter.*", }, { "source_labels": ["__meta_kubernetes_pod_annotation_prometheus_io_path"], @@ -1634,6 +1634,11 @@ def _collector_config( 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_USED"', 'scale_metric(1048576.0) where metric.name == "DCGM_FI_DEV_FB_FREE"', 'scale_metric(0.001) where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'scale_metric(1048576.0) where metric.name == "gpu_used_vram"', + 'scale_metric(1048576.0) where metric.name == "gpu_free_vram"', + 'scale_metric(0.01) where metric.name == "gpu_gfx_activity"', + 'scale_metric(0.01) where metric.name == "gpu_umc_activity"', + 'scale_metric(0.000001) where metric.name == "gpu_energy_consumed"', ], }, { @@ -1699,6 +1704,13 @@ def _collector_config( 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "DCGM_FI_DEV_GPU_TEMP"', 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "DCGM_FI_DEV_POWER_USAGE"', 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION"', + 'set(metric.name, "modelplane_gpu_memory_used_bytes") where metric.name == "gpu_used_vram"', + 'set(metric.name, "modelplane_gpu_memory_free_bytes") where metric.name == "gpu_free_vram"', + 'set(metric.name, "modelplane_gpu_compute_active_ratio") where metric.name == "gpu_gfx_activity"', + 'set(metric.name, "modelplane_gpu_memory_bandwidth_ratio") where metric.name == "gpu_umc_activity"', + 'set(metric.name, "modelplane_gpu_temperature_celsius") where metric.name == "gpu_junction_temperature"', + 'set(metric.name, "modelplane_gpu_power_watts") where metric.name == "gpu_power_usage"', + 'set(metric.name, "modelplane_energy_joules_total") where metric.name == "gpu_energy_consumed"', ], }, ] @@ -4641,7 +4653,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="522decbfdd3a589c", + config_hash="27157e659a3a8882", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -4999,7 +5011,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "collector": _collector( - config_hash="52b78175754428dd", + config_hash="159b3bf6c7617256", volumes=[ {"name": "config", "configMap": {"name": "modelplane-collector"}}, {"name": "credentials-primary", "secret": {"secretName": "telemetry-credentials"}}, @@ -5325,7 +5337,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="12df8b0b19eb5fce", + config_hash="c1024ab8dec03d88", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5627,7 +5639,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="46ed90b25528eb92", + config_hash="4cd9e55f075ccddc", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, @@ -5957,7 +5969,7 @@ def _to_dict(msg: message.Message) -> dict: service_extensions=None, ), "collector": _collector( - config_hash="522decbfdd3a589c", + config_hash="27157e659a3a8882", volumes=[{"name": "config", "configMap": {"name": "modelplane-collector"}}], volume_mounts=[{"name": "config", "mountPath": "/conf"}], env_from=None, diff --git a/functions/compose-serving-stack/tests/test_metrics.py b/functions/compose-serving-stack/tests/test_metrics.py index 7e79f22f7..47b1737b9 100644 --- a/functions/compose-serving-stack/tests/test_metrics.py +++ b/functions/compose-serving-stack/tests/test_metrics.py @@ -164,7 +164,11 @@ def test_mappings_scraped() -> None: """Each built-in mapping has a scrape of the component it reads.""" scraped = {case.mapping for case in PUBLISHED_CASES} builtin = {m.metadata.name for m in stacks.BUILTIN_MAPPINGS if m.metadata} - assert scraped == builtin, "a built-in mapping has no scrape to check it against" + # The AMD GPU mapping was written against the device-metrics-exporter's + # documented field list, with no AMD pool to scrape: when one is available, + # scrape its exporter into a fixture, add its PublishedCase, and take this + # exemption out. + assert scraped == builtin - {"modelplane-gpu-amd"}, "a built-in mapping has no scrape to check it against" def test_histogram_layouts() -> None: diff --git a/schemas/.lock.json b/schemas/.lock.json index 01adac440..a10da1342 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "cb6d31114c84ee9720f2dcd68e9eaa36227e2376f3399200723d86b305cdb4a1", + "fs://apis": "d01f2af97e8caa4721e70016af693002b4202ed9d46db666dc0cb88735f68e7d", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.8.1": "sha256:ca2e9e3b2e3a8b6ca44a9700d5abf7abd733cfa388d1afe9bb2bf7c847cd39ef", "xpkg://xpkg.upbound.io/upbound/provider-aws-efs:v2.8.1": "sha256:ebb1bcd8dc9a7e60e97a324652fbd9d1b0609ddbb1107db518ce999cbc1113bd", diff --git a/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py b/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py index f924b9700..253f2a6f0 100644 --- a/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/metricmapping/v1alpha1.py @@ -73,11 +73,18 @@ class Metric(BaseModel): A histogram is named by its base name, without the _count, _sum or _bucket a Prometheus query would use: the collector holds it as one metric, and `part` is what reaches into it. """ fromUnit: ( - Literal['Millijoules', 'Mebibytes', 'Milliseconds', 'Nanoseconds', 'Percent'] + Literal[ + 'Millijoules', + 'Microjoules', + 'Mebibytes', + 'Milliseconds', + 'Nanoseconds', + 'Percent', + ] | None ) = None """ - What the component measures this in, when that isn't the unit the name claims. Modelplane converts to the base unit: millijoules and milliseconds are divided by a thousand, nanoseconds by a billion, percent by a hundred, and mebibytes multiplied out to bytes. A histogram is converted whole - its sum, its bounds and its bucket boundaries - so its quantiles come out in the target unit too. + What the component measures this in, when that isn't the unit the name claims. Modelplane converts to the base unit: millijoules and milliseconds are divided by a thousand, microjoules by a million, nanoseconds by a billion, percent by a hundred, and mebibytes multiplied out to bytes. A histogram is converted whole - its sum, its bounds and its bucket boundaries - so its quantiles come out in the target unit too. Percent is for a component that counts a saturation from nought to a hundred where the name says a ratio. Check rather than assume: vLLM publishes kv_cache_usage_perc and the value is a fraction, so a name is no guide. """ labels: list[Label] | None = Field(None, max_length=16) From 0d8a59dcbe92990bd99f5023748e937e46cd40ae Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Thu, 22 Oct 2026 17:13:25 +0200 Subject: [PATCH 4/4] feat(docs): update for vale Signed-off-by: Christopher Haar --- .vale/styles/config/vocabularies/Modelplane/accept.txt | 6 ++++++ docs/content/recipes/qwen2.5-0.5b-cpu.md | 4 ++-- docs/content/recipes/qwen2.5-0.5b.md | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.vale/styles/config/vocabularies/Modelplane/accept.txt b/.vale/styles/config/vocabularies/Modelplane/accept.txt index b48c1c65b..cbcd03954 100644 --- a/.vale/styles/config/vocabularies/Modelplane/accept.txt +++ b/.vale/styles/config/vocabularies/Modelplane/accept.txt @@ -124,6 +124,9 @@ HBM3e cpuset device-metrics-exporter [Mm]icrojoules +MI325X +CPUs +NUMA Envoy Traefik MetalLB @@ -743,6 +746,9 @@ HBM3e cpuset device-metrics-exporter [Mm]icrojoules +MI325X +CPUs +NUMA Envoy Traefik MetalLB diff --git a/docs/content/recipes/qwen2.5-0.5b-cpu.md b/docs/content/recipes/qwen2.5-0.5b-cpu.md index f37ffba56..59828697a 100644 --- a/docs/content/recipes/qwen2.5-0.5b-cpu.md +++ b/docs/content/recipes/qwen2.5-0.5b-cpu.md @@ -22,10 +22,10 @@ Face. A CPU pool is the cheapest way to exercise a Modelplane deployment end to end - no GPU to find, no driver to wait for - and a 0.5B model is small enough that CPU inference answers in sensible time. -The class's provisioning block names no accelerator, which is what makes the +The class's provisioning block omits `accelerator`, which is what makes the pool a CPU pool: the cluster composition builds it with a standard node image and the `modelplane.ai/cpu` label and taint, and the serving stack installs -dra-driver-cpu - and no GPU machinery - because the class claims `dra.cpu`. +`dra-driver-cpu` - and no GPU machinery - because the class claims `dra.cpu`. The claim pins the engine to its NUMA node's cpuset. The engine is vLLM's CPU build, whose image serves the same OpenAI API as the CUDA one. The same class without its provisioning block describes a bring-your-own CPU pool, on any diff --git a/docs/content/recipes/qwen2.5-0.5b.md b/docs/content/recipes/qwen2.5-0.5b.md index 611a40a3f..1c5765f34 100644 --- a/docs/content/recipes/qwen2.5-0.5b.md +++ b/docs/content/recipes/qwen2.5-0.5b.md @@ -1,7 +1,7 @@ --- title: Qwen2.5-0.5B on AMD weight: 13 -description: A 0.5B dense chat model on a single AMD Instinct MI325X on a bring-your-own cluster. +description: A 0.5B dense chat model on one AMD Instinct MI325X on a bring-your-own cluster. model: Qwen/Qwen2.5-0.5B-Instruct vendors: [Qwen] clouds: [Existing] @@ -16,7 +16,7 @@ engineImages: [rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0. gpuNote: 1× of 8 per node --- -A 0.5B dense chat model on a single AMD Instinct MI325X on a bring-your-own +A 0.5B dense chat model on one AMD Instinct MI325X on a bring-your-own cluster: one `Standalone` engine, no cache, weights pulled straight from Hugging Face. The model is deliberately small - the point of this run is the AMD serving path, not the model. The class claims the GPU through DRA @@ -25,7 +25,7 @@ when a class claims that driver, against the amdgpu kernel driver the node image already carries. The engine is AMD's ROCm vLLM. The manifests are the ones validated end to end on an 8x MI325X bare metal -node (each GPU has just under 256 GiB of HBM3e), re-registered here as a +node (each GPU has just under 256 GiB of HBM3e), registered here as a bring-your-own cluster - `source: Existing` is the one cloud source that installs the AMD stack today. Bring the cluster with the amdgpu driver on its GPU nodes, label the pool's nodes `modelplane.ai/pool=gpu-mi325x`, and apply