feat: Fix webhook race condition kubernetes-sigs#1

Author: zhoubo9

pkg/manager/internal.go

@@ -591,16 +591,47 @@ func (cm *controllerManager) startNonLeaderElectionRunnables() {
 	cm.mu.Lock()
 	defer cm.mu.Unlock()
 
+	// First start any webhook servers, which includes conversion, validation, and defaulting
+	// webhooks that are registered.
+	//
+	// WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition
+	// between conversion webhooks and the cache sync (usually initial list) which causes the webhooks
+	// to never start because no cache can be populated.
+	for _, c := range cm.nonLeaderElectionRunnables {
+		if _, ok := c.(*webhook.Server); ok {
+			cm.startRunnable(c)
+		}
+	}
+
+	// Start and wait for caches.
 	cm.waitForCache(cm.internalCtx)
 
 	// Start the non-leaderelection Runnables after the cache has synced
 	for _, c := range cm.nonLeaderElectionRunnables {
+		if _, ok := c.(*webhook.Server); ok {
+			continue
+		}
+
 		// Controllers block, but we want to return an error if any have an error starting.
 		// Write any Start errors to a channel so we can return them
 		cm.startRunnable(c)
 	}
 }
 
+//func (cm *controllerManager) startNonLeaderElectionRunnables() {
+//	cm.mu.Lock()
+//	defer cm.mu.Unlock()
+//
+//	cm.waitForCache(cm.internalCtx)
+//
+//	// Start the non-leaderelection Runnables after the cache has synced
+//	for _, c := range cm.nonLeaderElectionRunnables {
+//		// Controllers block, but we want to return an error if any have an error starting.
+//		// Write any Start errors to a channel so we can return them
+//		cm.startRunnable(c)
+//	}
+//}
+
 func (cm *controllerManager) startLeaderElectionRunnables() {
 	cm.mu.Lock()
 	defer cm.mu.Unlock()