diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 25e501533..d6bb9d0af 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -1508,6 +1508,13 @@ contain the generated tokens. ### Upgrade notes +> **dcgm-exporter node selector:** the GKE accelerator label moved from the chart +> default to `values-gke.yaml`. Non-GKE releases upgraded with `--reuse-values` +> keep the old default selector; upgrade with `--reset-then-reuse-values` (Helm +> 3.14 or later) or pass your values with `-f`. +> GKE releases installed without `values-gke.yaml` must now pass it, or set +> `dcgm-exporter.nodeSelector`, or the exporter also runs on CPU nodes. + > **Installs without `config.auth.adminTokenSecretName`:** the upgrade creates > the admin-token Secret and adds `SIE_ADMIN_TOKEN` to sie-config, so it rolls. > sie-config then requires a token for every `/v1/configs` request, including diff --git a/deploy/helm/sie-cluster/values-aks.yaml b/deploy/helm/sie-cluster/values-aks.yaml index 849357da8..ac2ca17d3 100644 --- a/deploy/helm/sie-cluster/values-aks.yaml +++ b/deploy/helm/sie-cluster/values-aks.yaml @@ -178,5 +178,15 @@ serviceAccount: # azure.workload.identity/client-id: # inject via --set at deploy time dcgm-exporter: - # AKS GPU nodes use taints; no node selector required - nodeSelector: {} + # GPU node label set by the terraform-azure-sie GPU node pools + nodeSelector: + sie.superlinked.com/node-type: gpu + # Same taints as the GPU workers + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: kubernetes.azure.com/scalesetpriority + operator: Equal + value: spot + effect: NoSchedule diff --git a/deploy/helm/sie-cluster/values-aws.yaml b/deploy/helm/sie-cluster/values-aws.yaml index ac7772060..692c42c93 100644 --- a/deploy/helm/sie-cluster/values-aws.yaml +++ b/deploy/helm/sie-cluster/values-aws.yaml @@ -177,5 +177,6 @@ serviceAccount: # eks.amazonaws.com/role-arn: # inject via --set at deploy time dcgm-exporter: - # AWS GPU nodes use taints, not the GCP accelerator label - nodeSelector: {} + # GPU node label set by the terraform-aws-sie GPU node groups + nodeSelector: + sie.superlinked.com/node-type: gpu diff --git a/deploy/helm/sie-cluster/values.yaml b/deploy/helm/sie-cluster/values.yaml index 5c8b979a0..5ef3ddfe3 100644 --- a/deploy/helm/sie-cluster/values.yaml +++ b/deploy/helm/sie-cluster/values.yaml @@ -2124,11 +2124,8 @@ dcgm-exporter: interval: "15s" additionalLabels: app.kubernetes.io/part-of: sie - # -- Run on GPU nodes only. Override per cloud: - # GCP: cloud.google.com/gke-accelerator-count: "1" - # AWS: add dcgm-exporter.nodeSelector: {} in values-aws.yaml - nodeSelector: - cloud.google.com/gke-accelerator-count: "1" + # -- GPU node labels; set per cloud in the values-.yaml overlays. + nodeSelector: {} tolerations: - key: nvidia.com/gpu operator: Exists diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index c7b9fd22d..0debff876 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -4051,3 +4051,49 @@ def test_extra_volumes_cannot_reuse_a_chart_worker_volume_name(tmp_path: Path, n result = render_workers(tmp_path, extra_init_values(extraVolumes=[{"name": name, "emptyDir": {}}])) assert result.returncode != 0 assert f'workers.common.extraVolumes: "{name}" is a chart-owned worker volume name' in result.stderr + + +GKE_ACCELERATOR_LABEL = "cloud.google.com/gke-accelerator-count" + + +def dcgm_exporter_node_selector(tmp_path: Path, overlays: tuple[str, ...] = ()) -> dict: + docs = rendered_documents(tmp_path, {"dcgm-exporter": {"install": True}}, overlays) + (daemonset,) = [doc for doc in docs if doc["kind"] == "DaemonSet" and "dcgm-exporter" in doc["metadata"]["name"]] + return daemonset["spec"]["template"]["spec"].get("nodeSelector") or {} + + +@pytest.mark.parametrize( + ("overlay", "expected"), + [ + ("values-aws.yaml", {"sie.superlinked.com/node-type": "gpu"}), + ("values-aks.yaml", {"sie.superlinked.com/node-type": "gpu"}), + ( + "values-ack.yaml", + { + "ack.node.gpu.schedule": "default", + "sie.superlinked.com/gpu-type": "a10", + "sie.superlinked.com/node-type": "gpu", + }, + ), + ("values-gke.yaml", {GKE_ACCELERATOR_LABEL: "1"}), + ], +) +def test_dcgm_exporter_selects_only_the_cloud_overlay_gpu_labels(tmp_path: Path, overlay: str, expected: dict) -> None: + assert dcgm_exporter_node_selector(tmp_path, (overlay,)) == expected + + +def test_dcgm_exporter_has_no_cloud_specific_default_selector(tmp_path: Path) -> None: + assert dcgm_exporter_node_selector(tmp_path) == {} + + +def test_dcgm_exporter_tolerates_the_aks_spot_taint(tmp_path: Path) -> None: + docs = rendered_documents(tmp_path, {"dcgm-exporter": {"install": True}}, ("values-aks.yaml",)) + (daemonset,) = [doc for doc in docs if doc["kind"] == "DaemonSet" and "dcgm-exporter" in doc["metadata"]["name"]] + tolerations = daemonset["spec"]["template"]["spec"]["tolerations"] + assert {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} in tolerations + assert { + "key": "kubernetes.azure.com/scalesetpriority", + "operator": "Equal", + "value": "spot", + "effect": "NoSchedule", + } in tolerations