fix(helm): schedule dcgm-exporter on GPU nodes outside GKE

deploy/helm/sie-cluster/README.md

@@ -1508,6 +1508,13 @@ contain the generated tokens.
 
 ### Upgrade notes
 
+> **dcgm-exporter node selector:** the GKE accelerator label moved from the chart
+> default to `values-gke.yaml`. Non-GKE releases upgraded with `--reuse-values`
+> keep the old default selector; upgrade with `--reset-then-reuse-values` (Helm
+> 3.14 or later) or pass your values with `-f`.
+> GKE releases installed without `values-gke.yaml` must now pass it, or set
+> `dcgm-exporter.nodeSelector`, or the exporter also runs on CPU nodes.
+
 > **Installs without `config.auth.adminTokenSecretName`:** the upgrade creates
 > the admin-token Secret and adds `SIE_ADMIN_TOKEN` to sie-config, so it rolls.
 > sie-config then requires a token for every `/v1/configs` request, including

deploy/helm/sie-cluster/values-aks.yaml

@@ -178,5 +178,15 @@ serviceAccount:
   # azure.workload.identity/client-id: <client-id>  # inject via --set at deploy time
 
 dcgm-exporter:
-  # AKS GPU nodes use taints; no node selector required
-  nodeSelector: {}
+  # GPU node label set by the terraform-azure-sie GPU node pools
+  nodeSelector:
+    sie.superlinked.com/node-type: gpu
+  # Same taints as the GPU workers
+  tolerations:
+    - key: nvidia.com/gpu
+      operator: Exists
+      effect: NoSchedule
+    - key: kubernetes.azure.com/scalesetpriority
+      operator: Equal
+      value: spot
+      effect: NoSchedule

deploy/helm/sie-cluster/values-aws.yaml

@@ -177,5 +177,6 @@ serviceAccount:
   # eks.amazonaws.com/role-arn: <ARN>  # inject via --set at deploy time
 
 dcgm-exporter:
-  # AWS GPU nodes use taints, not the GCP accelerator label
-  nodeSelector: {}
+  # GPU node label set by the terraform-aws-sie GPU node groups
+  nodeSelector:
+    sie.superlinked.com/node-type: gpu

deploy/helm/sie-cluster/values.yaml

@@ -2124,11 +2124,8 @@ dcgm-exporter:
     interval: "15s"
     additionalLabels:
       app.kubernetes.io/part-of: sie
-  # -- Run on GPU nodes only. Override per cloud:
-  # GCP: cloud.google.com/gke-accelerator-count: "1"
-  # AWS: add dcgm-exporter.nodeSelector: {} in values-aws.yaml
-  nodeSelector:
-    cloud.google.com/gke-accelerator-count: "1"
+  # -- GPU node labels; set per cloud in the values-<cloud>.yaml overlays.
+  nodeSelector: {}
   tolerations:
     - key: nvidia.com/gpu
       operator: Exists

tools/ci/tests/test_helm_render.py

@@ -4051,3 +4051,49 @@ def test_extra_volumes_cannot_reuse_a_chart_worker_volume_name(tmp_path: Path, n
     result = render_workers(tmp_path, extra_init_values(extraVolumes=[{"name": name, "emptyDir": {}}]))
     assert result.returncode != 0
     assert f'workers.common.extraVolumes: "{name}" is a chart-owned worker volume name' in result.stderr
+
+
+GKE_ACCELERATOR_LABEL = "cloud.google.com/gke-accelerator-count"
+
+
+def dcgm_exporter_node_selector(tmp_path: Path, overlays: tuple[str, ...] = ()) -> dict:
+    docs = rendered_documents(tmp_path, {"dcgm-exporter": {"install": True}}, overlays)
+    (daemonset,) = [doc for doc in docs if doc["kind"] == "DaemonSet" and "dcgm-exporter" in doc["metadata"]["name"]]
+    return daemonset["spec"]["template"]["spec"].get("nodeSelector") or {}
+
+
+@pytest.mark.parametrize(
+    ("overlay", "expected"),
+    [
+        ("values-aws.yaml", {"sie.superlinked.com/node-type": "gpu"}),
+        ("values-aks.yaml", {"sie.superlinked.com/node-type": "gpu"}),
+        (
+            "values-ack.yaml",
+            {
+                "ack.node.gpu.schedule": "default",
+                "sie.superlinked.com/gpu-type": "a10",
+                "sie.superlinked.com/node-type": "gpu",
+            },
+        ),
+        ("values-gke.yaml", {GKE_ACCELERATOR_LABEL: "1"}),
+    ],
+)
+def test_dcgm_exporter_selects_only_the_cloud_overlay_gpu_labels(tmp_path: Path, overlay: str, expected: dict) -> None:
+    assert dcgm_exporter_node_selector(tmp_path, (overlay,)) == expected
+
+
+def test_dcgm_exporter_has_no_cloud_specific_default_selector(tmp_path: Path) -> None:
+    assert dcgm_exporter_node_selector(tmp_path) == {}
+
+
+def test_dcgm_exporter_tolerates_the_aks_spot_taint(tmp_path: Path) -> None:
+    docs = rendered_documents(tmp_path, {"dcgm-exporter": {"install": True}}, ("values-aks.yaml",))
+    (daemonset,) = [doc for doc in docs if doc["kind"] == "DaemonSet" and "dcgm-exporter" in doc["metadata"]["name"]]
+    tolerations = daemonset["spec"]["template"]["spec"]["tolerations"]
+    assert {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} in tolerations
+    assert {
+        "key": "kubernetes.azure.com/scalesetpriority",
+        "operator": "Equal",
+        "value": "spot",
+        "effect": "NoSchedule",
+    } in tolerations