Add a startup extension so main stops calling into pkg/enterprise

Makefile

@@ -169,10 +169,8 @@ EXCLUDE_MANIFEST_REGISTRIES?=gcr.io/
 # amd64 only. Constrain ARCHES (not just VALIDARCHES) so push-all, which iterates ARCHES, does not
 # try to push arches that were never built.
 ARCHES:=amd64
-# Bake cloud mode into the operator binary so it cannot be disabled at runtime (see isCloudBuild in
-# cmd/cloud.go). buildVariant lives in package main, which the linker addresses as "main" (not by its
-# import path), so this -X target is "main.buildVariant" rather than a $(PACKAGE_NAME)-prefixed path.
-CLOUD_LDFLAGS=-X main.buildVariant=cloud
+# Bake cloud mode into the operator binary so it cannot be disabled at runtime.
+CLOUD_LDFLAGS=-X $(PACKAGE_NAME)/version.BuildVariant=cloud
 endif
 
 BUILD_IMAGE?=tigera/operator

cmd/main.go

@@ -43,7 +43,6 @@ import (
 	"github.com/tigera/operator/pkg/controller/utils"
 	"github.com/tigera/operator/pkg/dns"
 	"github.com/tigera/operator/pkg/enterprise"
-	eoptions "github.com/tigera/operator/pkg/enterprise/options"
 	"github.com/tigera/operator/pkg/imports/admission"
 	"github.com/tigera/operator/pkg/imports/crds"
 	"github.com/tigera/operator/pkg/render"
@@ -87,14 +86,6 @@ var (
 // configuration for the operator loaded at startup.
 const bootstrapConfigMapName = "operator-bootstrap-config"
 
-// buildVariant is set to "cloud" via -ldflags "-X main.buildVariant=cloud" when building the Calico
-// Cloud operator image (see CLOUD_LDFLAGS in the Makefile), and is empty otherwise.
-var buildVariant string
-
-func isCloudBuild() bool {
-	return buildVariant == "cloud"
-}
-
 func init() {
 	// +kubebuilder:scaffold:scheme
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
@@ -104,8 +95,8 @@ func init() {
 
 func printVersion() {
 	log.Info(fmt.Sprintf("Version: %v", version.VERSION))
-	if isCloudBuild() {
-		log.Info("Variant: Calico Cloud")
+	if version.BuildVariant != "" {
+		log.Info(fmt.Sprintf("Variant: %s", version.BuildVariant))
 	}
 	log.Info(fmt.Sprintf("Go Version: %s", goruntime.Version()))
 	log.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goruntime.GOOS, goruntime.GOARCH))
@@ -191,8 +182,8 @@ admission policy installation; once an Installation exists it is the authority o
 		fmt.Println("Operator:", version.VERSION)
 		fmt.Println("Calico:", components.CalicoRelease)
 		fmt.Println("Enterprise:", components.EnterpriseRelease)
-		if isCloudBuild() {
-			fmt.Println("Variant: Calico Cloud")
+		if version.BuildVariant != "" {
+			fmt.Println("Variant:", version.BuildVariant)
 		}
 		os.Exit(0)
 	}
@@ -425,6 +416,32 @@ admission policy installation; once an Installation exists it is the authority o
 		}
 	}
 
+	clientset, err := kubernetes.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		log.Error(err, "Failed to get Kubernetes clientset")
+		os.Exit(1)
+	}
+
+	// Attempt to auto discover the provider
+	provider, err := discovery.AutoDiscoverProvider(ctx, clientset)
+	if err != nil {
+		setupLog.Error(err, "Auto discovery of Provider failed")
+		os.Exit(1)
+	}
+	setupLog.WithValues("provider", provider).Info("Checking type of cluster")
+
+	clusterDomain, err := dns.GetClusterDomain(dns.DefaultResolveConfPath)
+	if err != nil {
+		clusterDomain = dns.DefaultClusterDomain
+		log.Error(err, fmt.Sprintf("Couldn't find the cluster domain from the resolv.conf, defaulting to %s", clusterDomain))
+	}
+
+	kubernetesVersion, err := common.GetKubernetesVersion(clientset)
+	if err != nil {
+		log.Error(err, "Unable to resolve Kubernetes version, defaulting to v1.18")
+		kubernetesVersion = &common.VersionInfo{Major: 1, Minor: 18}
+	}
+
 	// Resolve the variant now that the operator CRDs exist.
 	variant := waitForVariant(ctx, c, setupLog)
 	setupLog.WithValues("variant", variant).Info("Resolved product variant")
@@ -439,9 +456,16 @@ admission policy installation; once an Installation exists it is the authority o
 		}
 	}
 
+	extensionRegistry, err := enterprise.Build(ctx, variant, clientset, manageCRDs, v3CRDs)
+	if err != nil {
+		setupLog.Error(err, "Failed to build the variant's extensions")
+		os.Exit(1)
+	}
+	setupLog.WithValues("tenancy", extensionRegistry.Startup().MultiTenant()).Info("Checking tenancy mode")
+
 	// The variant's controllers can't register without their APIs. Exiting lets the kubelet
 	// retry us once the CRDs are installed.
-	if err := enterprise.VerifyAPIsExist(variant, cs); err != nil {
+	if err := extensionRegistry.Startup().VerifyAPIsExist(cs); err != nil {
 		setupLog.Error(err, "Cannot run as the configured variant")
 		os.Exit(1)
 	}
@@ -508,45 +532,11 @@ admission policy installation; once an Installation exists it is the authority o
 		}
 	}()
 
-	clientset, err := kubernetes.NewForConfig(mgr.GetConfig())
-	if err != nil {
-		log.Error(err, "Failed to get Kubernetes clientset")
-		os.Exit(1)
-	}
-
-	// Attempt to auto discover the provider
-	provider, err := discovery.AutoDiscoverProvider(ctx, clientset)
-	if err != nil {
-		setupLog.Error(err, "Auto discovery of Provider failed")
-		os.Exit(1)
-	}
-	setupLog.WithValues("provider", provider).Info("Checking type of cluster")
-
-	// Determine if we're running in single or multi-tenant mode.
-	multiTenant, err := discovery.MultiTenant(ctx, clientset)
-	if err != nil {
-		log.Error(err, "Failed to discovery tenancy mode")
-		os.Exit(1)
-	}
-	setupLog.WithValues("tenancy", multiTenant).Info("Checking tenancy mode")
-
-	clusterDomain, err := dns.GetClusterDomain(dns.DefaultResolveConfPath)
-	if err != nil {
-		clusterDomain = dns.DefaultClusterDomain
-		log.Error(err, fmt.Sprintf("Couldn't find the cluster domain from the resolv.conf, defaulting to %s", clusterDomain))
-	}
-
-	kubernetesVersion, err := common.GetKubernetesVersion(clientset)
-	if err != nil {
-		log.Error(err, "Unable to resolve Kubernetes version, defaulting to v1.18")
-		kubernetesVersion = &common.VersionInfo{Major: 1, Minor: 18}
-	}
-
 	// The operator MUST not run within one of the Namespaces that it itself manages. Perform an early check here
 	// to make sure that we're not doing so, and exit if we are.
 	// Components share namespaces, so dedupe before the error lists them.
 	badNamespaces := sets.New(common.CalicoNamespace, render.CSIDaemonSetNamespace).
-		Insert(enterprise.ProtectedNamespaces()...).
+		Insert(extensionRegistry.Startup().ProtectedNamespaces()...).
 		UnsortedList()
 	for _, ns := range badNamespaces {
 		if common.OperatorNamespace() == ns {
@@ -569,11 +559,11 @@ admission policy installation; once an Installation exists it is the authority o
 	useSingleIndex := false
 	useExternalElastic := discovery.UseExternalElastic(bootConfig)
 
-	if isCloudBuild() {
+	if extensionRegistry.Startup().Cloud() {
 		elasticIsMigrating = discovery.ElasticIsMigrating(bootConfig)
 		useSingleIndex = discovery.UseSingleIndex(bootConfig)
-		if err := enterprise.VerifyElasticsearch(ctx, cs, variant, elasticIsMigrating, useExternalElastic); err != nil {
-			setupLog.Error(err, "Elasticsearch configuration verification failed")
+		if err := extensionRegistry.Startup().VerifyClusterState(ctx, cs, elasticIsMigrating, useExternalElastic); err != nil {
+			setupLog.Error(err, "Cluster state verification failed")
 			os.Exit(1)
 		}
 	}
@@ -590,14 +580,6 @@ admission policy installation; once an Installation exists it is the authority o
 		os.Exit(1)
 	}
 
-	// Build the extensions for the variant we resolved above.
-	extensionRegistry := enterprise.New(variant, eoptions.Options{
-		MultiTenant: multiTenant,
-		Cloud:       isCloudBuild(),
-		ManageCRDs:  manageCRDs,
-		UseV3CRDs:   v3CRDs,
-	})
-
 	options := options.ControllerOptions{
 		DetectedProvider:  provider,
 		Variant:           variant,
@@ -606,9 +588,9 @@ admission policy installation; once an Installation exists it is the authority o
 		ManageCRDs:        manageCRDs,
 		ShutdownContext:   ctx,
 		K8sClientset:      clientset,
-		MultiTenant:       multiTenant,
+		MultiTenant:       extensionRegistry.Startup().MultiTenant(),
 		ElasticExternal:   useExternalElastic,
-		Cloud:             isCloudBuild(),
+		Cloud:             extensionRegistry.Startup().Cloud(),
 		ESMigration:       elasticIsMigrating,
 		UseSingleIndex:    useSingleIndex,
 		UseV3CRDs:         v3CRDs,

pkg/enterprise/register.go

@@ -15,7 +15,13 @@
 package enterprise
 
 import (
+	"context"
+	"fmt"
+
+	"k8s.io/client-go/kubernetes"
+
 	operatorv1 "github.com/tigera/operator/api/v1"
+	"github.com/tigera/operator/pkg/common/discovery"
 	"github.com/tigera/operator/pkg/components"
 	"github.com/tigera/operator/pkg/enterprise/apiserver"
 	"github.com/tigera/operator/pkg/enterprise/clusterconnection"
@@ -29,39 +35,68 @@ import (
 	"github.com/tigera/operator/pkg/enterprise/whisker"
 	"github.com/tigera/operator/pkg/enterprise/windows"
 	"github.com/tigera/operator/pkg/extensions"
+	"github.com/tigera/operator/version"
 )
 
-// New builds the Calico Enterprise extensions. After the monorepo split this is what
-// calico-private's main constructs instead.
+// Build returns the Calico Enterprise extensions, and is called once the variant is
+// resolved. Tenancy and the cloud build flag are resolved here rather than passed in,
+// since each only ever describes an Enterprise install.
+func Build(ctx context.Context, variant operatorv1.ProductVariant, clientset kubernetes.Interface, manageCRDs, useV3CRDs bool) (extensions.Extensions, error) {
+	o := eoptions.Options{
+		ManageCRDs: manageCRDs,
+		UseV3CRDs:  useV3CRDs,
+		Cloud:      isCloudBuild(),
+	}
+
+	if variant.IsEnterprise() {
+		// Tenancy shows up as a namespaced Manager, a CRD only Enterprise installs.
+		multiTenant, err := discovery.MultiTenant(ctx, clientset)
+		if err != nil {
+			return extensions.Extensions{}, fmt.Errorf("failed to determine the tenancy mode: %w", err)
+		}
+		o.MultiTenant = multiTenant
+	}
+
+	return New(variant, o), nil
+}
+
+// New builds the Calico Enterprise extensions from options already resolved.
 func New(variant operatorv1.ProductVariant, o eoptions.Options) extensions.Extensions {
+	// Startup is registered whatever the variant, since the namespaces Enterprise
+	// manages are off limits to a Calico install too.
+	set := extensions.Set{Startup: startup{variant: variant, opts: o}}
+
 	// Enterprise has two spellings, so match on the product rather than the constant:
 	// an Installation asking for the deprecated TigeraSecureEnterprise still gets the
 	// Enterprise extensions.
-	if variant.IsEnterprise() {
+	switch {
+	case variant.IsEnterprise():
 		// Registered here so the images arrive with the extensions. A test that wants
 		// this build's own images instead calls components.UseImages.
 		components.RegisterVariantImages(components.EnterpriseImages)
 
-		return extensions.New(extensions.Set{
-			Installation:      installation.New(variant, o),
-			Windows:           windows.New(variant),
-			APIServer:         apiserver.New(variant, o),
-			ClusterConnection: clusterconnection.New(variant),
-			Tiers:             tiers.New(o),
-			CSR:               csr.New(),
-			Istio:             istio.New(variant),
-			Goldmane:          goldmane.New(variant),
-			Whisker:           whisker.New(variant),
-			GatewayAPI:        gatewayapi.New(variant),
-		})
-	}
-
-	if variant == operatorv1.Calico {
+		set.Installation = installation.New(variant, o)
+		set.Windows = windows.New(variant)
+		set.APIServer = apiserver.New(variant, o)
+		set.ClusterConnection = clusterconnection.New(variant)
+		set.Tiers = tiers.New(o)
+		set.CSR = csr.New()
+		set.Istio = istio.New(variant)
+		set.Goldmane = goldmane.New(variant)
+		set.Whisker = whisker.New(variant)
+		set.GatewayAPI = gatewayapi.New(variant)
+	case variant == operatorv1.Calico:
 		// Clean up what a prior Enterprise installation left behind.
-		return extensions.New(extensions.Set{APIServer: apiserver.CalicoCleanup{}})
+		set.APIServer = apiserver.CalicoCleanup{}
 	}
 
-	return extensions.Extensions{}
+	return extensions.New(set)
+}
+
+// isCloudBuild reports whether the binary was built for Calico Cloud. The Makefile bakes
+// the answer in through CLOUD_LDFLAGS so it cannot be flipped at runtime.
+func isCloudBuild() bool {
+	return version.BuildVariant == "cloud"
 }
 
 // Images is the image set Enterprise runs, for the caller to register. New leaves it

pkg/enterprise/startup.go

@@ -24,16 +24,25 @@ import (
 
 	operatorv1 "github.com/tigera/operator/api/v1"
 	"github.com/tigera/operator/pkg/common/discovery"
+	eoptions "github.com/tigera/operator/pkg/enterprise/options"
 	"github.com/tigera/operator/pkg/render"
 	"github.com/tigera/operator/pkg/render/intrusiondetection/dpi"
 	"github.com/tigera/operator/pkg/render/istio"
 	"github.com/tigera/operator/pkg/render/logstorage"
 	"github.com/tigera/operator/pkg/render/logstorage/eck"
 )
 
+// startup is the Enterprise hook into operator startup. It registers for every variant,
+// since the namespaces Enterprise manages are off limits to a Calico install too, and
+// no-ops the checks that only apply to Enterprise.
+type startup struct {
+	variant operatorv1.ProductVariant
+	opts    eoptions.Options
+}
+
 // VerifyAPIsExist reports whether the Enterprise CRDs the extension controllers need are installed.
-func VerifyAPIsExist(variant operatorv1.ProductVariant, cs kubernetes.Interface) error {
-	if !variant.IsEnterprise() {
+func (s startup) VerifyAPIsExist(cs kubernetes.Interface) error {
+	if !s.variant.IsEnterprise() {
 		return nil
 	}
 
@@ -47,10 +56,10 @@ func VerifyAPIsExist(variant operatorv1.ProductVariant, cs kubernetes.Interface)
 	return nil
 }
 
-// VerifyElasticsearch rejects a cluster whose Elasticsearch certificates contradict the
+// VerifyClusterState rejects a cluster whose Elasticsearch certificates contradict the
 // internal or external mode the operator is configured for.
-func VerifyElasticsearch(ctx context.Context, cs kubernetes.Interface, variant operatorv1.ProductVariant, migrating, external bool) error {
-	if !variant.IsEnterprise() {
+func (s startup) VerifyClusterState(ctx context.Context, cs kubernetes.Interface, migrating, external bool) error {
+	if !s.variant.IsEnterprise() {
 		return nil
 	}
 
@@ -80,7 +89,7 @@ func VerifyElasticsearch(ctx context.Context, cs kubernetes.Interface, variant o
 
 // ProtectedNamespaces returns the Enterprise namespaces the operator manages and so
 // must not run in itself.
-func ProtectedNamespaces() []string {
+func (s startup) ProtectedNamespaces() []string {
 	return []string{
 		render.ElasticsearchNamespace,
 		render.IntrusionDetectionNamespace,
@@ -91,3 +100,13 @@ func ProtectedNamespaces() []string {
 		istio.IstioNamespace,
 	}
 }
+
+// MultiTenant reports the tenancy mode resolved at startup.
+func (s startup) MultiTenant() bool {
+	return s.opts.MultiTenant
+}
+
+// Cloud reports whether this binary was built for Calico Cloud.
+func (s startup) Cloud() bool {
+	return s.opts.Cloud
+}