feat(ksm): grant DRA RBAC for Cluster Agent 7.85.0+; mount CDI dir for GPU monitoring

config/rbac/role.yaml

@@ -612,6 +612,15 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - resource.k8s.io
+  resources:
+  - devicetaintrules
+  - resourceclaims
+  - resourceslices
+  verbs:
+  - list
+  - watch
 - apiGroups:
   - security.openshift.io
   resourceNames:

docs/kubernetes_state_metrics.md

@@ -142,6 +142,41 @@ The above will have the operator create and maintain a ConfigMap for you with th
 
 NB: You can't use `configData` and `configMap` simultaneously.
 
+### Dynamic Resource Allocation (DRA) metrics
+
+To collect metrics about Kubernetes [Dynamic Resource Allocation][dra] objects, add the `resourceclaims`, `resourceslices`, and `devicetaintrules` collectors to a custom check configuration (`conf.configData` or `conf.configMap`). A custom `collectors` list replaces the default one, so also list the default collectors you want to keep (see the default list above):
+
+```yaml
+apiVersion: datadoghq.com/v2alpha1
+kind: DatadogAgent
+metadata:
+  name: datadog
+spec:
+  features:
+    kubeStateMetricsCore:
+      enabled: true
+      conf:
+        configData: |-
+          cluster_check: false
+          init_config:
+          instances:
+            - skip_leader_election: false
+              collectors:
+              # ... default collectors you want to keep ...
+              - resourceclaims
+              - resourceslices
+              - devicetaintrules
+```
+
+This example runs the check on the Cluster Agent. To run it on Cluster Checks Runners instead, set `features.clusterChecks.enabled: true` and `features.clusterChecks.useClusterChecksRunners: true`, then use `cluster_check: true` and `skip_leader_election: true`; the Cluster Checks Runner image must be 7.85.0 or later.
+
+The DRA collectors require Cluster Agent 7.85.0 or later (or Cluster Checks Runner 7.85.0 or later when the check runs on Cluster Checks Runners). Older versions fail the whole `kubernetes_state_core` check when these collectors are listed on a cluster that serves the DRA API. When the component running the check is 7.85.0 or later, the operator automatically grants `list` and `watch` on `resourceclaims`, `resourceslices`, and `devicetaintrules` in the `resource.k8s.io` API group. Resource claims and resource slices require Kubernetes 1.34 or later, and device taint rules need the DeviceTaintRule API (stable in Kubernetes 1.37, earlier behind a feature gate); the Agent skips collectors whose API the cluster does not serve.
+
+For a complete setup that also enables GPU monitoring and pins the Agent and Cluster Agent to 7.85.0, see [datadog-agent-with-dra.yaml][dra-example].
+
+[dra]: https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/
+[dra-example]: https://github.com/DataDog/datadog-operator/blob/main/examples/datadogagent/datadog-agent-with-dra.yaml
+
 ## Further Reading
 
 The v2 of the Kubernetes State Metrics check is embedded as a "core check" in the Datadog Agent.

examples/datadogagent/datadog-agent-with-dra.yaml

@@ -0,0 +1,72 @@
+apiVersion: datadoghq.com/v2alpha1
+kind: DatadogAgent
+metadata:
+  name: datadog
+spec:
+  features:
+    # Mounts the host CDI spec directory so the Agent can attribute GPUs allocated through DRA to pods.
+    gpu:
+      enabled: true
+    kubeStateMetricsCore:
+      enabled: true
+      conf:
+        # configData replaces the default collector list, so it repeats the full default list
+        # and adds the DRA collectors at the end.
+        configData: |-
+          cluster_check: false
+          init_config:
+          instances:
+            - skip_leader_election: false
+              # Optional: only count claims requesting these DeviceClasses.
+              # dra_device_classes:
+              #   - gpu.nvidia.com
+              collectors:
+              - pods
+              - replicationcontrollers
+              - statefulsets
+              - nodes
+              - cronjobs
+              - jobs
+              - replicasets
+              - deployments
+              - configmaps
+              - services
+              - endpoints
+              - endpointslices
+              - daemonsets
+              - horizontalpodautoscalers
+              - poddisruptionbudgets
+              - limitranges
+              - resourcequotas
+              - secrets
+              - namespaces
+              - persistentvolumeclaims
+              - persistentvolumes
+              - ingresses
+              - storageclasses
+              - volumeattachments
+              # Add verticalpodautoscalers if the VPA CRDs are installed.
+              - apiservices
+              - controllerrevisions
+              - customresourcedefinitions
+              # DRA collectors: require Cluster Agent 7.85.0+.
+              # resourceclaims and resourceslices need Kubernetes 1.34+; devicetaintrules needs the DeviceTaintRule API
+              # (stable in Kubernetes 1.37, earlier behind a feature gate);
+              # the Agent skips collectors whose API the cluster does not serve.
+              - resourceclaims
+              - resourceslices
+              - devicetaintrules
+  global:
+    credentials:
+      apiKey: <DATADOG_API_KEY>
+      appKey: <DATADOG_APP_KEY>
+    clusterName: foo
+  override:
+    # The operator grants the resource.k8s.io RBAC only for Cluster Agent 7.85.0+.
+    clusterAgent:
+      image:
+        tag: "7.85.0"
+    # GPU attribution for DRA pods needs Agent 7.85.0+.
+    nodeAgent:
+      image:
+        tag: "7.85.0"

internal/controller/datadogagent/feature/gpu/const.go

@@ -18,6 +18,12 @@ const (
 	gkeCOSNVIDIADriverLib64HostPath   = "/home/kubernetes/bin/nvidia/lib64"
 	gkeCOSNVIDIADriverLib64MountPath  = "/host/run/nvidia/driver/usr/lib/x86_64-linux-gnu"
 
+	// CDI specs written by DRA drivers on the host; the agent reads them to
+	// attribute GPUs to pods that obtained them through DRA resource claims.
+	cdiVolumeName = "cdi"
+	cdiHostPath   = "/var/run/cdi"
+	cdiMountPath  = "/host/var/run/cdi"
+
 	// defaultGPURuntimeClass default runtime class for GPU pods
 	defaultGPURuntimeClass = "nvidia"
 )

internal/controller/datadogagent/feature/gpu/feature.go

@@ -49,6 +49,7 @@ func (f *gpuFeature) ID() feature.IDType {
 // by the nvidia-container-runtime; mount them from the host location.
 // On Talos, tracefs is a standalone mount rather than nested under the debugfs
 // mount privileged mode adds, so system-probe needs it explicitly.
+// On GKE Autopilot, the CDI hostPath is not in the WorkloadAllowlist, so it is removed.
 func (f *gpuFeature) NodeAgentProviderCapabilities() providercaps.ProviderCapabilityMap {
 	containers := []apicommon.AgentContainerName{apicommon.CoreAgentContainerName}
 	if f.isPrivilegedModeEnabled {
@@ -68,6 +69,9 @@ func (f *gpuFeature) NodeAgentProviderCapabilities() providercaps.ProviderCapabi
 				},
 			},
 		},
+		kubernetes.GKEAutopilotProvider: {
+			RemoveVolumes: []string{cdiVolumeName},
+		},
 	}
 
 	if f.isPrivilegedModeEnabled {
@@ -316,6 +320,13 @@ func (f *gpuFeature) ManageNodeAgent(managers feature.PodTemplateManagers) error
 	managers.Volume().AddVolume(&devicesVol)
 	managers.VolumeMount().AddVolumeMountToContainer(&devicesMount, apicommon.CoreAgentContainerName)
 
+	// Mount the host CDI spec directory so the agent can attribute GPUs allocated
+	// through DRA resource claims. DirectoryOrCreate so a missing dir never blocks pod start.
+	cdiVol, cdiMount := volume.GetVolumes(cdiVolumeName, cdiHostPath, cdiMountPath, true)
+	cdiVol.HostPath.Type = ptr.To(corev1.HostPathDirectoryOrCreate)
+	managers.Volume().AddVolume(&cdiVol)
+	managers.VolumeMount().AddVolumeMountToContainer(&cdiMount, apicommon.CoreAgentContainerName)
+
 	// Configure the runtime class for the pod
 	if f.podRuntimeClassName != "" {
 		managers.PodTemplateSpec().Spec.RuntimeClassName = &f.podRuntimeClassName

internal/controller/datadogagent/feature/gpu/feature_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/DataDog/datadog-operator/internal/controller/datadogagent/feature"
 	"github.com/DataDog/datadog-operator/internal/controller/datadogagent/feature/fake"
 	"github.com/DataDog/datadog-operator/internal/controller/datadogagent/feature/test"
+	"github.com/DataDog/datadog-operator/internal/controller/datadogagent/object/volume"
 	"github.com/DataDog/datadog-operator/internal/controller/datadogagent/override"
 	"github.com/DataDog/datadog-operator/internal/controller/datadogagent/providercaps"
 	"github.com/DataDog/datadog-operator/pkg/kubernetes"
@@ -94,6 +95,11 @@ func Test_GPUMonitoringFeature_Configure(t *testing.T) {
 				MountPath: nvidiaDevicesMountPath,
 				ReadOnly:  true,
 			},
+			{
+				Name:      cdiVolumeName,
+				MountPath: cdiMountPath,
+				ReadOnly:  true,
+			},
 			{
 				Name:      common.KubeletPodResourcesVolumeName,
 				MountPath: podResourcesSocketPath,
@@ -175,6 +181,15 @@ func Test_GPUMonitoringFeature_Configure(t *testing.T) {
 					},
 				},
 			},
+			{
+				Name: cdiVolumeName,
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: cdiHostPath,
+						Type: ptr.To(corev1.HostPathDirectoryOrCreate),
+					},
+				},
+			},
 			{
 				Name: common.KubeletPodResourcesVolumeName,
 				VolumeSource: corev1.VolumeSource{
@@ -268,6 +283,11 @@ func Test_GPUMonitoringFeature_Configure(t *testing.T) {
 				MountPath: nvidiaDevicesMountPath,
 				ReadOnly:  true,
 			},
+			{
+				Name:      cdiVolumeName,
+				MountPath: cdiMountPath,
+				ReadOnly:  true,
+			},
 			{
 				Name:      common.KubeletPodResourcesVolumeName,
 				MountPath: podResourcesSocketPath,
@@ -292,6 +312,15 @@ func Test_GPUMonitoringFeature_Configure(t *testing.T) {
 					},
 				},
 			},
+			{
+				Name: cdiVolumeName,
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: cdiHostPath,
+						Type: ptr.To(corev1.HostPathDirectoryOrCreate),
+					},
+				},
+			},
 			{
 				Name: common.KubeletPodResourcesVolumeName,
 				VolumeSource: corev1.VolumeSource{
@@ -494,6 +523,34 @@ func Test_GPUFeature_NodeAgentProviderCapabilities(t *testing.T) {
 		assert.NotContains(t, getContainer(tmpl, apicommon.SystemProbeContainerName).VolumeMounts, wantMount)
 	})
 
+	t.Run("gke-autopilot removes the CDI volume and mount", func(t *testing.T) {
+		f := &gpuFeature{}
+		tmpl := newPodTemplate()
+		mgr := feature.NewPodTemplateManagers(tmpl)
+		cdiVol, cdiMount := volume.GetVolumes(cdiVolumeName, cdiHostPath, cdiMountPath, true)
+		mgr.Volume().AddVolume(&cdiVol)
+		mgr.VolumeMount().AddVolumeMountToContainer(&cdiMount, apicommon.CoreAgentContainerName)
+
+		providercaps.ApplyProviderCapabilities(mgr, kubernetes.GKEAutopilotProvider, f.NodeAgentProviderCapabilities())
+
+		assert.Empty(t, tmpl.Spec.Volumes)
+		assert.Empty(t, getContainer(tmpl, apicommon.CoreAgentContainerName).VolumeMounts)
+	})
+
+	t.Run("non-autopilot provider keeps the CDI volume and mount", func(t *testing.T) {
+		f := &gpuFeature{}
+		tmpl := newPodTemplate()
+		mgr := feature.NewPodTemplateManagers(tmpl)
+		cdiVol, cdiMount := volume.GetVolumes(cdiVolumeName, cdiHostPath, cdiMountPath, true)
+		mgr.Volume().AddVolume(&cdiVol)
+		mgr.VolumeMount().AddVolumeMountToContainer(&cdiMount, apicommon.CoreAgentContainerName)
+
+		providercaps.ApplyProviderCapabilities(mgr, kubernetes.DefaultProvider, f.NodeAgentProviderCapabilities())
+
+		assert.Contains(t, tmpl.Spec.Volumes, cdiVol)
+		assert.Contains(t, getContainer(tmpl, apicommon.CoreAgentContainerName).VolumeMounts, cdiMount)
+	})
+
 	t.Run("non-gke-cos provider adds nothing", func(t *testing.T) {
 		f := &gpuFeature{isPrivilegedModeEnabled: true}
 		tmpl := newPodTemplate()

internal/controller/datadogagent/feature/kubernetesstatecore/const.go

@@ -36,6 +36,12 @@ const (
 	// Sharding only coexists safely with the cluster_aggregates_only
 	// collection mode as of https://github.com/DataDog/datadog-agent/pull/54126.
 	ksmShardingMinVersion = "7.83.0-0"
+
+	// Minimum agent version that supports the DRA collectors (resourceclaims,
+	// resourceslices, devicetaintrules). Older agents fail the whole check when
+	// asked for resourceclaims on clusters serving the DRA API, so the DRA RBAC
+	// rule is only granted from this version.
+	draCollectorsMinVersion = "7.85.0-0"
 )
 
 // GetKubeStateMetricsRBACResourceName return the RBAC resources name

internal/controller/datadogagent/feature/kubernetesstatecore/feature.go

@@ -67,6 +67,10 @@ type ksmFeature struct {
 
 	shardingEnabled bool
 
+	// draRBAC is true when the component running the check supports the DRA
+	// collectors, so users can list them in a custom KSM config.
+	draRBAC bool
+
 	rbacSuffix         string
 	serviceAccountName string
 
@@ -245,6 +249,17 @@ func (f *ksmFeature) Configure(dda metav1.Object, ddaSpec *v2alpha1.DatadogAgent
 			}
 		}
 
+		// Grant read access to the DRA API so users can add the resourceclaims,
+		// resourceslices and devicetaintrules collectors in a custom config.
+		// Fail closed: older or unparseable versions of the component running the check
+		// (Cluster Agent or Cluster Checks Runners) do not get the rule.
+		draComponent := v2alpha1.ClusterAgentComponentName
+		if f.runInClusterChecksRunner {
+			draComponent = v2alpha1.ClusterChecksRunnerComponentName
+		}
+		draFallback := false
+		f.draRBAC = utils.IsAboveMinVersion(common.GetComponentVersion(dda, draComponent), draCollectorsMinVersion, &draFallback)
+
 		f.configConfigMapName = constants.GetConfName(dda, f.customConfig, defaultKubeStateMetricsCoreConf)
 	}
 
@@ -257,6 +272,7 @@ type collectorOptions struct {
 	enableCRD                 bool
 	enableControllerRevisions bool
 	useAPIServerCache         bool
+	enableDRARBAC             bool
 	customResources           []v2alpha1.Resource
 }
 
@@ -272,6 +288,7 @@ func (f *ksmFeature) ManageDependencies(managers feature.ResourceManagers) error
 		enableCRD:                 f.collectCRDMetrics,
 		enableControllerRevisions: f.collectControllerRevisions,
 		useAPIServerCache:         f.useAPIServerCache,
+		enableDRARBAC:             f.draRBAC,
 		customResources:           f.collectCrMetrics,
 	}
 	configCM, err := f.buildKSMCoreConfigMap(collectorOpts)