Attribute DCGM exporter metrics to pods on DRA nodes by default

internal/state/dcgm_exporter.go

@@ -131,7 +131,6 @@ func buildDCGMExporterRenderData(ctx context.Context, s *configurableState, cr *
 		Collectors:                   collectors,
 		HPCJobMappingDir:             hpcJobMappingDir,
 		PodLabelAllowlistRegex:       strings.Join(spec.PodLabelAllowlistRegex, ","),
-		PodMetadataEnabled:           spec.IsKubernetesPodMetadataEnabled(),
 		EnablePodLabels:              spec.IsPodLabelsEnabled(),
 		EnablePodUID:                 spec.IsPodUIDEnabled(),
 		HostPID:                      spec.IsHostPIDEnabled(),

internal/state/dcgm_exporter_test.go

@@ -80,8 +80,10 @@ func TestDCGMExporterEnabledByDefault(t *testing.T) {
 	assert.Equal(t, 1, kinds["ResourceClaimTemplate"])
 	assert.Equal(t, 1, kinds["DaemonSet"])
 	assert.Equal(t, 1, kinds["Service"])
-	// No pod-read ClusterRole and no ServiceMonitor by default.
-	assert.Equal(t, 0, kinds["ClusterRole"])
+	// DRA attribution needs the ResourceSlice informer, so the read ClusterRole is
+	// always bound. No ServiceMonitor by default.
+	assert.Equal(t, 1, kinds["ClusterRole"])
+	assert.Equal(t, 1, kinds["ClusterRoleBinding"])
 	assert.Equal(t, 0, kinds["ServiceMonitor"])
 
 	claimHasAdminAccess(t, findByKind(objs, "ResourceClaimTemplate"))
@@ -90,12 +92,16 @@ func TestDCGMExporterEnabledByDefault(t *testing.T) {
 	podSpec := ds.Spec.Template.Spec
 	assert.Equal(t, "true", podSpec.NodeSelector["nvidia.com/gpu.deploy.dcgm-exporter-dra"])
 	require.NotNil(t, podSpec.AutomountServiceAccountToken)
-	assert.False(t, *podSpec.AutomountServiceAccountToken)
+	assert.True(t, *podSpec.AutomountServiceAccountToken)
 
 	ctr := podSpec.Containers[0]
 	assert.Equal(t, "nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3", ctr.Image)
 	env := envMap(ctr.Env)
 	assert.Equal(t, ":9400", env["DCGM_EXPORTER_LISTEN"])
+	// Pod attribution on DRA nodes is on by default; pod labels and UID are not.
+	assert.Equal(t, "true", env["KUBERNETES_ENABLE_DRA"])
+	assert.NotContains(t, env, "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS")
+	assert.NotContains(t, env, "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID")
 	assert.Equal(t, "/etc/dcgm-exporter/dcp-metrics-included.csv", env["DCGM_EXPORTER_COLLECTORS"])
 	// Embedded engine: no remote host-engine env.
 	_, hasRemote := env["DCGM_REMOTE_HOSTENGINE_INFO"]
@@ -138,16 +144,8 @@ func TestDCGMExporterPodMetadataEnrichment(t *testing.T) {
 	objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog())
 	require.NoError(t, err)
 
-	// The pod-read ClusterRole + binding render only when enrichment is on.
-	kinds := kindCounts(objs)
-	assert.Equal(t, 1, kinds["ClusterRole"])
-	assert.Equal(t, 1, kinds["ClusterRoleBinding"])
-
 	ds := findDaemonSet(t, objs)
 	podSpec := ds.Spec.Template.Spec
-	// Enrichment needs a mounted SA token.
-	require.NotNil(t, podSpec.AutomountServiceAccountToken)
-	assert.True(t, *podSpec.AutomountServiceAccountToken)
 	env := envMap(podSpec.Containers[0].Env)
 	assert.Equal(t, "true", env["DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS"])
 	assert.Equal(t, "true", env["DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID"])

internal/state/gpucluster_render_test.go

@@ -131,8 +131,8 @@ func TestGPUClusterRenderGolden(t *testing.T) {
 			},
 		},
 		{
-			// enablePodLabels: pod metadata mounts the ServiceAccount token and adds
-			// the DRA/pod-labels env and the read-pods ClusterRole/Binding.
+			// enablePodLabels adds the pod-labels env on top of the pod attribution
+			// that every DRA exporter gets.
 			name: "gpucluster-dcgm-exporter-pod-metadata",
 			render: func(t *testing.T) []*unstructured.Unstructured {
 				s := newTestDCGMExporterState(t, false)

internal/state/testdata/golden/gpucluster-dcgm-exporter-embedded.yaml

@@ -20,6 +20,30 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: nvidia-dcgm-exporter-dra
+  name: nvidia-dcgm-exporter-dra-read-pods
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - resource.k8s.io
+  resources:
+  - resourceslices
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: nvidia-dcgm-exporter-dra
@@ -33,6 +57,21 @@ subjects:
   name: nvidia-dcgm-exporter-dra
   namespace: test-operator
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: nvidia-dcgm-exporter-dra
+  name: nvidia-dcgm-exporter-dra-read-pods
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nvidia-dcgm-exporter-dra-read-pods
+subjects:
+- kind: ServiceAccount
+  name: nvidia-dcgm-exporter-dra
+  namespace: test-operator
+---
 apiVersion: resource.k8s.io/v1
 kind: ResourceClaimTemplate
 metadata:
@@ -83,7 +122,7 @@ spec:
       labels:
         app: nvidia-dcgm-exporter-dra
     spec:
-      automountServiceAccountToken: false
+      automountServiceAccountToken: true
       containers:
       - env:
         - name: DCGM_EXPORTER_LISTEN
@@ -96,6 +135,8 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        - name: KUBERNETES_ENABLE_DRA
+          value: "true"
         image: nvcr.io/nvidia/k8s/dcgm-exporter:test
         livenessProbe:
           httpGet:

internal/state/testdata/golden/gpucluster-dcgm-exporter-remote-engine.yaml

@@ -20,6 +20,30 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: nvidia-dcgm-exporter-dra
+  name: nvidia-dcgm-exporter-dra-read-pods
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - resource.k8s.io
+  resources:
+  - resourceslices
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: nvidia-dcgm-exporter-dra
@@ -33,6 +57,21 @@ subjects:
   name: nvidia-dcgm-exporter-dra
   namespace: test-operator
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: nvidia-dcgm-exporter-dra
+  name: nvidia-dcgm-exporter-dra-read-pods
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nvidia-dcgm-exporter-dra-read-pods
+subjects:
+- kind: ServiceAccount
+  name: nvidia-dcgm-exporter-dra
+  namespace: test-operator
+---
 apiVersion: resource.k8s.io/v1
 kind: ResourceClaimTemplate
 metadata:
@@ -83,7 +122,7 @@ spec:
       labels:
         app: nvidia-dcgm-exporter-dra
     spec:
-      automountServiceAccountToken: false
+      automountServiceAccountToken: true
       containers:
       - env:
         - name: DCGM_EXPORTER_LISTEN
@@ -98,6 +137,8 @@ spec:
               fieldPath: spec.nodeName
         - name: DCGM_REMOTE_HOSTENGINE_INFO
           value: nvidia-dcgm-dra:5555
+        - name: KUBERNETES_ENABLE_DRA
+          value: "true"
         image: nvcr.io/nvidia/k8s/dcgm-exporter:test
         livenessProbe:
           httpGet:

internal/state/types.go

@@ -97,13 +97,11 @@ type dcgmExporterRenderData struct {
 	// on vanilla Kubernetes.
 	OpenshiftVersion string
 	// ResourceClaimAPIVersion is the apiVersion to render on ResourceClaimTemplate objects.
-	ResourceClaimAPIVersion string
-	RemoteHostEngine        string
-	Collectors              string
-	HPCJobMappingDir        string
-	PodLabelAllowlistRegex  string
-	// PodMetadataEnabled mounts the ServiceAccount token and binds the pods-read ClusterRole.
-	PodMetadataEnabled           bool
+	ResourceClaimAPIVersion      string
+	RemoteHostEngine             string
+	Collectors                   string
+	HPCJobMappingDir             string
+	PodLabelAllowlistRegex       string
 	EnablePodLabels              bool
 	EnablePodUID                 bool
 	HostPID                      bool

manifests/state-dcgm-exporter/0210_clusterrole.yaml

@@ -1,4 +1,3 @@
-{{- if .PodMetadataEnabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -22,4 +21,3 @@ rules:
   - get
   - list
   - watch
-{{- end }}

manifests/state-dcgm-exporter/0310_clusterrolebinding.yaml

@@ -1,4 +1,3 @@
-{{- if .PodMetadataEnabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -13,4 +12,3 @@ subjects:
 - kind: ServiceAccount
   name: nvidia-dcgm-exporter-dra
   namespace: {{ .Namespace }}
-{{- end }}

manifests/state-dcgm-exporter/0700_daemonset.yaml

@@ -41,7 +41,7 @@ spec:
     spec:
       priorityClassName: system-node-critical
       serviceAccountName: nvidia-dcgm-exporter-dra
-      automountServiceAccountToken: {{ .PodMetadataEnabled }}
+      automountServiceAccountToken: true
       # Gate scheduling on the per-component deploy label so the k8s-driver-manager
       # can pause it to drain dcgm-exporter off a node during a driver reload.
       nodeSelector:
@@ -99,10 +99,11 @@ spec:
         - name: DCGM_HPC_JOB_MAPPING_DIR
           value: {{ .HPCJobMappingDir | quote }}
         {{- end }}
-        {{- if .PodMetadataEnabled }}
+        # The kubelet reports DRA-allocated devices as (pool, device) names rather
+        # than GPU UUIDs, so the exporter needs the ResourceSlice informer to resolve
+        # them. Without this the metrics carry no pod, namespace or container label.
         - name: KUBERNETES_ENABLE_DRA
           value: "true"
-        {{- end }}
         {{- if .EnablePodLabels }}
         - name: DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS
           value: "true"