Remove mode label and do not allow ClusterPolicy & GPUCluster co-existence

controllers/active_config.go

@@ -24,7 +24,6 @@ import (
 
 	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
 	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
-	"github.com/NVIDIA/gpu-operator/internal/consts"
 )
 
 // getSingletonClusterPolicy returns the ClusterPolicy treated as the cluster-wide
@@ -72,25 +71,3 @@ func resolveActiveConfig(ctx context.Context, c client.Reader) (*gpuv1.ClusterPo
 
 	return getSingletonClusterPolicy(clusterPolicies.Items), gpuCluster, nil
 }
-
-// resolveDefaultMode returns the nvidia.com/gpu-operator.resource-allocation.mode value for a GPU node that
-// does not have one yet. When exactly one configuration CR exists its stack wins;
-// envDefaultMode (the validated DEFAULT_GPU_ALLOCATION_MODE operator environment variable)
-// is consulted only when both CRs exist, defaulting to device-plugin when unset. Nodes
-// already labeled are never touched, so changing DEFAULT_GPU_ALLOCATION_MODE only affects
-// nodes labeled afterward.
-func resolveDefaultMode(clusterPolicyExists, gpuClusterExists bool, envDefaultMode consts.GPUAllocationMode) consts.GPUAllocationMode {
-	switch {
-	case clusterPolicyExists && gpuClusterExists:
-		if envDefaultMode == consts.GPUAllocationModeDRA {
-			return consts.GPUAllocationModeDRA
-		}
-		return consts.GPUAllocationModeDevicePlugin
-	case gpuClusterExists:
-		return consts.GPUAllocationModeDRA
-	case clusterPolicyExists:
-		return consts.GPUAllocationModeDevicePlugin
-	default:
-		return ""
-	}
-}

controllers/active_config_test.go

@@ -32,7 +32,6 @@ import (
 
 	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
 	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
-	"github.com/NVIDIA/gpu-operator/internal/consts"
 )
 
 func TestResolveActiveConfig(t *testing.T) {
@@ -137,26 +136,3 @@ func TestResolveActiveConfig(t *testing.T) {
 		assert.Nil(t, gc)
 	})
 }
-
-func TestResolveDefaultMode(t *testing.T) {
-	testCases := []struct {
-		description         string
-		clusterPolicyExists bool
-		gpuClusterExists    bool
-		envDefaultMode      consts.GPUAllocationMode
-		expected            consts.GPUAllocationMode
-	}{
-		{"both CRs, DEFAULT_GPU_ALLOCATION_MODE=dra", true, true, consts.GPUAllocationModeDRA, consts.GPUAllocationModeDRA},
-		{"both CRs, DEFAULT_GPU_ALLOCATION_MODE=device-plugin", true, true, consts.GPUAllocationModeDevicePlugin, consts.GPUAllocationModeDevicePlugin},
-		{"both CRs, DEFAULT_GPU_ALLOCATION_MODE unset defaults to device-plugin", true, true, "", consts.GPUAllocationModeDevicePlugin},
-		{"only ClusterPolicy ignores DEFAULT_GPU_ALLOCATION_MODE", true, false, consts.GPUAllocationModeDRA, consts.GPUAllocationModeDevicePlugin},
-		{"only GPUCluster ignores DEFAULT_GPU_ALLOCATION_MODE", false, true, consts.GPUAllocationModeDevicePlugin, consts.GPUAllocationModeDRA},
-		{"neither CR resolves to no mode", false, false, consts.GPUAllocationModeDRA, ""},
-	}
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			mode := resolveDefaultMode(tc.clusterPolicyExists, tc.gpuClusterExists, tc.envDefaultMode)
-			assert.Equal(t, tc.expected, mode)
-		})
-	}
-}

controllers/clusterpolicy_controller.go

@@ -131,8 +131,25 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, nil
 	}
 
+	// TODO: remove the below code block once both ClusterPolicy and GPUCluster can co-exist
+	gpuClusters := &nvidiav1alpha1.GPUClusterList{}
+	if err := r.List(ctx, gpuClusters); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to list GPUCluster objects: %w", err)
+	}
+	if len(gpuClusters.Items) > 0 {
+		err := fmt.Errorf("conflicting GPUCluster resource %q detected; ClusterPolicy and GPUCluster cannot co-exist", gpuClusters.Items[0].Name)
+		r.Log.Error(err, "only one CR may be present at a time")
+		updateCRState(ctx, r, req.NamespacedName, gpuv1.NotReady)
+		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
+			r.Log.Error(condErr, "failed to set condition")
+		}
+		clusterPolicyCtrl.operatorMetrics.reconciliationStatus.Set(reconciliationStatusClusterPolicyUnavailable)
+		return ctrl.Result{}, err
+	}
+
 	if err := clusterPolicyCtrl.init(ctx, r, instance); err != nil {
 		r.Log.Error(err, "unable to initialize ClusterPolicy controller")
+		updateCRState(ctx, r, req.NamespacedName, gpuv1.NotReady)
 		if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
 			r.Log.Error(condErr, "failed to set condition")
 		}
@@ -366,16 +383,12 @@ func addWatchNewGPUNode(r *ClusterPolicyReconciler, c controller.Controller, mgr
 			newOSTreeLabel := newLabels[nfdOSTreeVersionLabelKey]
 			osTreeLabelChanged := oldOSTreeLabel != newOSTreeLabel
 
-			// The resource-allocation mode label gates rendering of the mode nodeSelector
-			// on operand DaemonSets, so re-render when it lands or changes.
-			modeLabelChanged := oldLabels[consts.GPUAllocationModeLabelKey] != newLabels[consts.GPUAllocationModeLabelKey]
 			driverOwnerLabelChanged, driverUpgradeStateLabelChanged, driverUpgradeSkipLabelChanged := driverUpgradeLabelsChanged(oldLabels, newLabels)
 
 			needsUpdate := gpuCommonLabelAdded ||
 				commonOperandsLabelChanged ||
 				gpuWorkloadConfigLabelChanged ||
 				osTreeLabelChanged ||
-				modeLabelChanged ||
 				driverOwnerLabelChanged ||
 				driverUpgradeStateLabelChanged ||
 				driverUpgradeSkipLabelChanged
@@ -387,7 +400,6 @@ func addWatchNewGPUNode(r *ClusterPolicyReconciler, c controller.Controller, mgr
 					"commonOperandsLabelChanged", commonOperandsLabelChanged,
 					"gpuWorkloadConfigLabelChanged", gpuWorkloadConfigLabelChanged,
 					"osTreeLabelChanged", osTreeLabelChanged,
-					"modeLabelChanged", modeLabelChanged,
 					"driverOwnerLabelChanged", driverOwnerLabelChanged,
 					"driverUpgradeStateLabelChanged", driverUpgradeStateLabelChanged,
 					"driverUpgradeSkipLabelChanged", driverUpgradeSkipLabelChanged,

controllers/clusterpolicy_controller_test.go

@@ -444,6 +444,24 @@ func TestClusterPolicyReconcileSkipsNonSingleton(t *testing.T) {
 	require.Equal(t, gpuv1.Ready, clusterPolicyState(t, c, older.Name))
 }
 
+func TestClusterPolicyBlockedByGPUCluster(t *testing.T) {
+	cp := clusterPolicyForUpgradeTest(true)
+	r, c, _ := newClusterPolicyUpgradeTestReconciler(t, cp)
+
+	gc := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "config"}}
+	require.NoError(t, c.Create(t.Context(), gc))
+
+	_, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(cp)})
+	require.Equal(t, gpuv1.NotReady, clusterPolicyState(t, c, cp.Name))
+	require.ErrorContains(t, err, "ClusterPolicy and GPUCluster cannot co-exist")
+
+	// Deleting the GPUCluster instance unblocks the next reconcile
+	require.NoError(t, c.Delete(t.Context(), gc))
+	_, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(cp)})
+	require.NoError(t, err)
+	require.Equal(t, gpuv1.Ready, clusterPolicyState(t, c, cp.Name))
+}
+
 func newClusterPolicyUpgradeTestReconciler(t *testing.T, cp *gpuv1.ClusterPolicy, nodes ...*corev1.Node) (*ClusterPolicyReconciler, client.Client, *OperatorMetrics) {
 	t.Helper()
 	scheme := runtime.NewScheme()

controllers/gpucluster_controller.go

@@ -100,15 +100,6 @@ func (r *GPUClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, fmt.Errorf("error adding finalizer to GPUCluster %s: %w", req.NamespacedName, err)
 	}
 
-	// GPUCluster (DRA stack) may coexist with a ClusterPolicy (device-plugin
-	// stack): every operand DaemonSet of both stacks gates on the per-node
-	// nvidia.com/gpu-operator.resource-allocation.mode label, so each node is served by exactly one stack.
-
-	// No singleton claim is needed: the CRD's CEL rule pins metadata.name, so at most
-	// one GPUCluster can exist.
-
-	// DRA requires all driver management through NVIDIADriver CRs: surface an unmet
-	// prerequisite on this CR's status and hold off deploying operands until it is met.
 	if msg, err := r.validatePrerequisites(ctx); err != nil {
 		return ctrl.Result{}, err
 	} else if msg != "" {
@@ -171,10 +162,9 @@ func (r *GPUClusterReconciler) validatePrerequisites(ctx context.Context) (strin
 	if err := r.List(ctx, clusterPolicies); err != nil {
 		return "", fmt.Errorf("error listing ClusterPolicy objects: %w", err)
 	}
-	// Only the active singleton ClusterPolicy matters here: an Ignored instance
-	// deploys nothing, so it cannot own driver daemonsets.
-	if active := getSingletonClusterPolicy(clusterPolicies.Items); active != nil && !active.Spec.Driver.UseNvidiaDriverCRDType() {
-		return fmt.Sprintf("ClusterPolicy %s does not have driver.useNvidiaDriverCRD enabled; migrate driver management to NVIDIADriver CRs before enabling DRA", active.Name), nil
+	// TODO: relax this prerequisite once ClusterPolicy and GPUCluster can co-exist
+	if clusterPolicy := getSingletonClusterPolicy(clusterPolicies.Items); clusterPolicy != nil {
+		return fmt.Sprintf("A ClusterPolicy CR %q exists; a ClusterPolicy CR and GPUCluster CR may not exist at the same time", clusterPolicy.Name), nil
 	}
 	return "", nil
 }

controllers/gpucluster_controller_test.go

@@ -20,13 +20,11 @@ import (
 	"context"
 	"sort"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -38,7 +36,6 @@ import (
 
 	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
 	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
-	"github.com/NVIDIA/gpu-operator/internal/conditions"
 	"github.com/NVIDIA/gpu-operator/internal/state"
 )
 
@@ -217,52 +214,17 @@ func TestGPUClusterTeardownDrainsClaimConsumersFirst(t *testing.T) {
 	require.NoError(t, c.Get(t.Context(), types.NamespacedName{Name: plugin.Name, Namespace: "test-namespace"}, ds))
 }
 
-// A ClusterPolicy in the cluster does not disable the GPUCluster, provided it
-// delegates driver management to NVIDIADriver CRs: the two stacks coexist, with
-// per-node ownership decided by the nvidia.com/gpu-operator.resource-allocation.mode label.
+// A ClusterPolicy and GPUCluster CR cannot co-exist
 func TestGPUClusterCoexistsWithClusterPolicy(t *testing.T) {
-	cfg := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "config"}}
-	cp := &gpuv1.ClusterPolicy{
-		ObjectMeta: metav1.ObjectMeta{Name: "cluster-policy"},
-		Spec: gpuv1.ClusterPolicySpec{
-			Driver: gpuv1.DriverSpec{UseNvidiaDriverCRD: ptr.To(true)},
-		},
-	}
-	r, c := newGPUClusterReconciler(t, cfg, cp)
-
-	gccReconcile(t, r, cfg.Name)
-
-	require.Equal(t, nvidiav1alpha1.Ready, gccState(t, c, cfg.Name))
-}
-
-// A ClusterPolicy that manages its own driver (useNvidiaDriverCRD=false) is an invalid
-// companion for DRA: the GPUCluster reports the unmet prerequisite and deploys nothing.
-func TestGPUClusterClusterPolicyDriverPrerequisite(t *testing.T) {
 	cfg := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "config"}}
 	cp := &gpuv1.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{Name: "cluster-policy"}}
 	r, c := newGPUClusterReconciler(t, cfg, cp)
-	r.conditionUpdater = conditions.NewGPUClusterUpdater(c)
-
-	res, err := r.Reconcile(t.Context(), gccRequest(cfg.Name))
-	require.NoError(t, err)
-	require.Equal(t, time.Minute, res.RequeueAfter)
 
+	gccReconcile(t, r, cfg.Name)
 	require.Equal(t, nvidiav1alpha1.NotReady, gccState(t, c, cfg.Name))
-	require.Nil(t, r.stateManager.(*fakeStateManager).lastCatalog, "operands must not be synced")
-
-	instance := &nvidiav1alpha1.GPUCluster{}
-	require.NoError(t, c.Get(t.Context(), types.NamespacedName{Name: cfg.Name}, instance))
-	cond := meta.FindStatusCondition(instance.Status.Conditions, conditions.Error)
-	require.NotNil(t, cond)
-	require.Equal(t, conditions.PrerequisiteNotMet, cond.Reason)
-	require.Contains(t, cond.Message, "useNvidiaDriverCRD")
-
-	// Toggling the flag to true clears the prerequisite on the next reconcile.
-	updated := &gpuv1.ClusterPolicy{}
-	require.NoError(t, c.Get(t.Context(), types.NamespacedName{Name: cp.Name}, updated))
-	updated.Spec.Driver.UseNvidiaDriverCRD = ptr.To(true)
-	require.NoError(t, c.Update(t.Context(), updated))
 
+	// Deleting the ClusterPolicy instance satisfies the prerequisites on the next reconcile
+	require.NoError(t, c.Delete(t.Context(), cp))
 	gccReconcile(t, r, cfg.Name)
 	require.Equal(t, nvidiav1alpha1.Ready, gccState(t, c, cfg.Name))
 }