Mt framework

cmd/glbc/main.go

@@ -27,14 +27,11 @@ import (
 
 	firewallcrclient "github.com/GoogleCloudPlatform/gke-networking-api/client/gcpfirewall/clientset/versioned"
 	networkclient "github.com/GoogleCloudPlatform/gke-networking-api/client/network/clientset/versioned"
-	informernetwork "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
 	nodetopologyclient "github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/clientset/versioned"
-	informernodetopology "github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/informers/externalversions"
 	k8scp "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
 	flag "github.com/spf13/pflag"
 	crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	informers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection"
@@ -45,7 +42,7 @@ import (
 	frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned"
 	"k8s.io/ingress-gce/pkg/instancegroups"
 	"k8s.io/ingress-gce/pkg/l4lb"
-	multiprojectgce "k8s.io/ingress-gce/pkg/multiproject/gce"
+	multiprojectgce "k8s.io/ingress-gce/pkg/multiproject/common/gce"
 	multiprojectstart "k8s.io/ingress-gce/pkg/multiproject/start"
 	"k8s.io/ingress-gce/pkg/network"
 	providerconfigclient "k8s.io/ingress-gce/pkg/providerconfig/client/clientset/versioned"
@@ -54,7 +51,6 @@ import (
 	serviceattachmentclient "k8s.io/ingress-gce/pkg/serviceattachment/client/clientset/versioned"
 	"k8s.io/ingress-gce/pkg/svcneg"
 	svcnegclient "k8s.io/ingress-gce/pkg/svcneg/client/clientset/versioned"
-	informersvcneg "k8s.io/ingress-gce/pkg/svcneg/client/informers/externalversions"
 	"k8s.io/ingress-gce/pkg/systemhealth"
 	"k8s.io/ingress-gce/pkg/utils"
 	"k8s.io/klog/v2"
@@ -259,19 +255,6 @@ func main() {
 			if err != nil {
 				klog.Fatalf("Failed to create ProviderConfig client: %v", err)
 			}
-			informersFactory := informers.NewSharedInformerFactory(kubeClient, flags.F.ResyncPeriod)
-			var svcNegFactory informersvcneg.SharedInformerFactory
-			if svcNegClient != nil {
-				svcNegFactory = informersvcneg.NewSharedInformerFactory(svcNegClient, flags.F.ResyncPeriod)
-			}
-			var networkFactory informernetwork.SharedInformerFactory
-			if networkClient != nil {
-				networkFactory = informernetwork.NewSharedInformerFactory(networkClient, flags.F.ResyncPeriod)
-			}
-			var nodeTopologyFactory informernodetopology.SharedInformerFactory
-			if nodeTopologyClient != nil {
-				nodeTopologyFactory = informernodetopology.NewSharedInformerFactory(nodeTopologyClient, flags.F.ResyncPeriod)
-			}
 			ctx := context.Background()
 			syncerMetrics := syncMetrics.NewNegMetricsCollector(flags.F.NegMetricsExportInterval, rootLogger)
 			go syncerMetrics.Run(stopCh)
@@ -284,13 +267,11 @@ func main() {
 					rootLogger,
 					kubeClient,
 					svcNegClient,
+					networkClient,
+					nodeTopologyClient,
 					kubeSystemUID,
 					eventRecorderKubeClient,
 					providerConfigClient,
-					informersFactory,
-					svcNegFactory,
-					networkFactory,
-					nodeTopologyFactory,
 					gceCreator,
 					namer,
 					stopCh,
@@ -305,13 +286,11 @@ func main() {
 					rootLogger,
 					kubeClient,
 					svcNegClient,
+					networkClient,
+					nodeTopologyClient,
 					kubeSystemUID,
 					eventRecorderKubeClient,
 					providerConfigClient,
-					informersFactory,
-					svcNegFactory,
-					networkFactory,
-					nodeTopologyFactory,
 					gceCreator,
 					namer,
 					stopCh,

pkg/multiproject/README.md

@@ -0,0 +1,82 @@
+# Multi-Project Controller
+
+Enables ingress-gce to manage GCP resources across multiple projects through ProviderConfig CRs.
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                     Main Controller                          │
+│                                                              │
+│  ┌────────────────────────────────────────────────────────┐ │
+│  │              Shared Kubernetes Informers                │ │
+│  │         (Services, Ingresses, EndpointSlices)           │ │
+│  └─────────────────────┬──────────────────────────────────┘ │
+│                        │                                     │
+│  ┌─────────────────────▼──────────────────────────────────┐ │
+│  │           ProviderConfig Controller                     │ │
+│  │         Watches ProviderConfig resources                │ │
+│  │         Manages per-project controllers                 │ │
+│  └─────────────────────┬──────────────────────────────────┘ │
+│                        │                                     │
+│       ┌────────────────┼────────────────┐                   │
+│       │                │                │                   │
+│  ┌────▼─────┐   ┌─────▼─────┐   ┌─────▼─────┐             │
+│  │Project A │   │ Project B │   │ Project C │   ...        │
+│  │Controller│   │Controller │   │Controller │              │
+│  └──────────┘   └───────────┘   └───────────┘              │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Key Concepts
+
+- **ProviderConfig**: CR defining a GCP project configuration
+- **Resource Filtering**: Resources are associated via labels; each controller sees only its labeled resources
+- **Shared Informers**: Base informers are created once and shared; controllers get filtered views
+- **Dynamic Lifecycle**: Controllers start/stop with ProviderConfig create/delete
+
+## Usage
+
+### Create ProviderConfig
+
+```yaml
+apiVersion: networking.gke.io/v1
+kind: ProviderConfig
+metadata:
+  name: team-a-project
+spec:
+  projectID: team-a-gcp-project
+  network: team-a-network
+```
+
+### Associate Resources
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-service
+  labels:
+    ${PROVIDER_CONFIG_LABEL_KEY}: provider-config-a
+spec:
+  # Service spec...
+```
+
+## Operations
+
+### Adding a Project
+1. Create ProviderConfig
+2. Label services/ingresses with PC name
+3. NEGs created in target project
+
+### Removing a Project
+1. Remove/relabel services using the PC
+2. Wait for NEG cleanup
+3. Delete ProviderConfig
+
+## Guarantees
+
+- Controllers only manage explicitly labeled resources
+- One controller per ProviderConfig
+- Base infrastructure survives individual controller failures
+- PC deletion doesn't affect other projects

pkg/multiproject/finalizer/finalizer.go → pkg/multiproject/common/finalizer/finalizer.go

@@ -16,14 +16,15 @@ const (
 )
 
 func EnsureProviderConfigNEGCleanupFinalizer(cs *providerconfig.ProviderConfig, csClient providerconfigclient.Interface, logger klog.Logger) error {
-	return ensureProviderConfigFinalizer(cs, ProviderConfigNEGCleanupFinalizer, csClient, logger)
+	return EnsureProviderConfigFinalizer(cs, ProviderConfigNEGCleanupFinalizer, csClient, logger)
 }
 
 func DeleteProviderConfigNEGCleanupFinalizer(cs *providerconfig.ProviderConfig, csClient providerconfigclient.Interface, logger klog.Logger) error {
-	return deleteProviderConfigFinalizer(cs, ProviderConfigNEGCleanupFinalizer, csClient, logger)
+	return DeleteProviderConfigFinalizer(cs, ProviderConfigNEGCleanupFinalizer, csClient, logger)
 }
 
-func ensureProviderConfigFinalizer(pc *providerconfig.ProviderConfig, key string, csClient providerconfigclient.Interface, logger klog.Logger) error {
+// EnsureProviderConfigFinalizer ensures a finalizer with the given name is present on the ProviderConfig.
+func EnsureProviderConfigFinalizer(pc *providerconfig.ProviderConfig, key string, csClient providerconfigclient.Interface, logger klog.Logger) error {
 	if HasGivenFinalizer(pc.ObjectMeta, key) {
 		return nil
 	}
@@ -36,7 +37,8 @@ func ensureProviderConfigFinalizer(pc *providerconfig.ProviderConfig, key string
 	return patch.PatchProviderConfigObjectMetadata(csClient, pc, *updatedObjectMeta)
 }
 
-func deleteProviderConfigFinalizer(pc *providerconfig.ProviderConfig, key string, csClient providerconfigclient.Interface, logger klog.Logger) error {
+// DeleteProviderConfigFinalizer removes a finalizer with the given name from the ProviderConfig.
+func DeleteProviderConfigFinalizer(pc *providerconfig.ProviderConfig, key string, csClient providerconfigclient.Interface, logger klog.Logger) error {
 	if !HasGivenFinalizer(pc.ObjectMeta, key) {
 		return nil
 	}

pkg/multiproject/controller/controller.go → pkg/multiproject/framework/controller.go

@@ -1,8 +1,7 @@
-// Package controller implements the ProviderConfig controller that starts and stops controllers for each ProviderConfig.
-package controller
+// Package framework implements the ProviderConfig controller that starts and stops controllers for each ProviderConfig.
+package framework
 
 import (
-	"context"
 	"fmt"
 	"math/rand"
 	"runtime/debug"
@@ -18,17 +17,17 @@ const (
 	workersNum                   = 5
 )
 
-// ProviderConfigControllerManager implements the logic for starting and stopping controllers for each ProviderConfig.
-type ProviderConfigControllerManager interface {
+// providerConfigControllerManager implements the logic for starting and stopping controllers for each ProviderConfig.
+type providerConfigControllerManager interface {
 	StartControllersForProviderConfig(pc *providerconfig.ProviderConfig) error
 	StopControllersForProviderConfig(pc *providerconfig.ProviderConfig)
 }
 
-// ProviderConfigController is a controller that manages the ProviderConfig resource.
+// providerConfigController is a controller that manages the ProviderConfig resource.
 // It is responsible for starting and stopping controllers for each ProviderConfig.
-// Currently, it only manages the NEG controller using the ProviderConfigControllerManager.
-type ProviderConfigController struct {
-	manager ProviderConfigControllerManager
+// Currently, it only manages the NEG controller using the providerConfigControllerManager.
+type providerConfigController struct {
+	manager providerConfigControllerManager
 
 	providerConfigLister cache.Indexer
 	providerConfigQueue  utils.TaskQueue
@@ -38,10 +37,10 @@ type ProviderConfigController struct {
 	hasSynced            func() bool
 }
 
-// NewProviderConfigController creates a new instance of the ProviderConfig controller.
-func NewProviderConfigController(manager ProviderConfigControllerManager, providerConfigInformer cache.SharedIndexInformer, stopCh <-chan struct{}, logger klog.Logger) *ProviderConfigController {
+// newProviderConfigController creates a new instance of the ProviderConfig controller.
+func newProviderConfigController(manager providerConfigControllerManager, providerConfigInformer cache.SharedIndexInformer, stopCh <-chan struct{}, logger klog.Logger) *providerConfigController {
 	logger = logger.WithName(providerConfigControllerName)
-	pcc := &ProviderConfigController{
+	pcc := &providerConfigController{
 		providerConfigLister: providerConfigInformer.GetIndexer(),
 		stopCh:               stopCh,
 		numWorkers:           workersNum,
@@ -68,21 +67,16 @@ func NewProviderConfigController(manager ProviderConfigControllerManager, provid
 	return pcc
 }
 
-func (pcc *ProviderConfigController) Run() {
+func (pcc *providerConfigController) Run() {
 	defer pcc.shutdown()
 
 	pcc.logger.Info("Starting ProviderConfig controller")
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		<-pcc.stopCh
-		pcc.logger.Info("Stop channel closed, cancelling context")
-		cancel()
-	}()
 
 	pcc.logger.Info("Waiting for initial cache sync before starting ProviderConfig Controller")
-	ok := cache.WaitForCacheSync(ctx.Done(), pcc.hasSynced)
+	ok := cache.WaitForCacheSync(pcc.stopCh, pcc.hasSynced)
 	if !ok {
 		pcc.logger.Error(nil, "Failed to wait for initial cache sync before starting ProviderConfig Controller")
+		return
 	}
 
 	pcc.logger.Info("Started ProviderConfig Controller", "numWorkers", pcc.numWorkers)
@@ -91,12 +85,12 @@ func (pcc *ProviderConfigController) Run() {
 	<-pcc.stopCh
 }
 
-func (pcc *ProviderConfigController) shutdown() {
+func (pcc *providerConfigController) shutdown() {
 	pcc.logger.Info("Shutting down ProviderConfig Controller")
 	pcc.providerConfigQueue.Shutdown()
 }
 
-func (pcc *ProviderConfigController) syncWrapper(key string) error {
+func (pcc *providerConfigController) syncWrapper(key string) error {
 	syncID := rand.Int31()
 	svcLogger := pcc.logger.WithValues("providerConfigKey", key, "syncId", syncID)
 
@@ -113,7 +107,7 @@ func (pcc *ProviderConfigController) syncWrapper(key string) error {
 	return err
 }
 
-func (pcc *ProviderConfigController) sync(key string, logger klog.Logger) error {
+func (pcc *providerConfigController) sync(key string, logger klog.Logger) error {
 	logger = logger.WithName("providerConfig.sync")
 
 	providerConfig, exists, err := pcc.providerConfigLister.GetByKey(key)

pkg/multiproject/framework/controller_map.go

@@ -0,0 +1,79 @@
+package framework
+
+import "sync"
+
+// controllerSet holds controller-specific resources for a ProviderConfig.
+// It contains the stop channel used to signal controller shutdown and a mutex
+// that serializes lifecycle transitions for the underlying controllers.
+type controllerSet struct {
+	mu     sync.Mutex
+	stopCh chan<- struct{}
+}
+
+// ControllerMap is a thread-safe map for storing controllerSet instances.
+// It uses read-write locking to allow concurrent read operations.
+type ControllerMap struct {
+	mu   sync.RWMutex
+	data map[string]*controllerSet
+}
+
+// NewControllerMap creates a new thread-safe ControllerMap.
+func NewControllerMap() *ControllerMap {
+	return &ControllerMap{
+		data: make(map[string]*controllerSet),
+	}
+}
+
+// Get retrieves a controllerSet for the given key.
+// Returns the controllerSet and a boolean indicating whether it exists.
+func (cm *ControllerMap) Get(key string) (*controllerSet, bool) {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+	cs, exists := cm.data[key]
+	return cs, exists
+}
+
+// GetOrCreate retrieves the controllerSet for the given key, creating a new entry when absent.
+// The second return value indicates whether the controllerSet already existed.
+func (cm *ControllerMap) GetOrCreate(key string) (*controllerSet, bool) {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	if cs, exists := cm.data[key]; exists {
+		return cs, true
+	}
+	cs := &controllerSet{}
+	cm.data[key] = cs
+	return cs, false
+}
+
+// Set stores a controllerSet for the given key.
+func (cm *ControllerMap) Set(key string, cs *controllerSet) {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	cm.data[key] = cs
+}
+
+// Delete removes and returns a controllerSet for the given key.
+// Returns the controllerSet and a boolean indicating whether it existed.
+func (cm *ControllerMap) Delete(key string) (*controllerSet, bool) {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	cs, exists := cm.data[key]
+	if exists {
+		delete(cm.data, key)
+	}
+	return cs, exists
+}
+
+// DeleteIfSame removes the controllerSet for key only if it currently equals expected.
+// Returns true if a deletion happened.
+func (cm *ControllerMap) DeleteIfSame(key string, expected *controllerSet) bool {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	cur, ok := cm.data[key]
+	if !ok || cur != expected {
+		return false
+	}
+	delete(cm.data, key)
+	return true
+}