Revert "Enable leader election in ws-manager-mk2"

components/ws-manager-mk2/cmd/sample-workspace/main.go

@@ -10,11 +10,10 @@ import (
 	"log"
 	"time"
 
+	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/yaml"
-
-	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
 )
 
 func main() {

components/ws-manager-mk2/config/manager/manager.yaml

@@ -33,6 +33,8 @@ spec:
       containers:
       - command:
         - /manager
+        args:
+        - --leader-elect
         image: controller:latest
         name: manager
         securityContext:

components/ws-manager-mk2/controllers/maintenance_controller.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
 	"sync"
 	"time"
 
@@ -18,12 +17,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 var (
@@ -112,46 +106,9 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti
 	log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil)
 }
 
-func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
-	// We need to use an unmanaged controller to avoid issues when the pod is in standby mode.
-	// In that scenario, the controllers are not started and don't watch changes and only
-	// observe the maintenance mode during the initialization.
-	c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r})
-	if err != nil {
-		return err
-	}
-
-	go func() {
-		err = c.Start(ctx)
-		if err != nil {
-			log.FromContext(ctx).Error(err, "cannot start maintenance reconciler")
-			os.Exit(1)
-		}
-	}()
-
-	return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{})
-}
-
-type filterConfigMap struct {
-	predicate.Funcs
-}
-
-func (f filterConfigMap) Create(e event.CreateEvent) bool {
-	return f.filter(e.Object)
-}
-
-func (f filterConfigMap) Update(e event.UpdateEvent) bool {
-	return f.filter(e.ObjectNew)
-}
-
-func (f filterConfigMap) Generic(e event.GenericEvent) bool {
-	return f.filter(e.Object)
-}
-
-func (f filterConfigMap) filter(obj client.Object) bool {
-	if obj == nil {
-		return false
-	}
-
-	return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace
+func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("maintenance").
+		For(&corev1.ConfigMap{}).
+		Complete(r)
 }

components/ws-manager-mk2/main.go

@@ -11,15 +11,13 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	"k8s.io/client-go/rest"
 
 	"github.com/bombsimon/logrusr/v2"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
@@ -70,9 +68,13 @@ func init() {
 }
 
 func main() {
+	var enableLeaderElection bool
 	var configFN string
 	var jsonLog bool
 	var verbose bool
+	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
+		"Enable leader election for controller manager. "+
+			"Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&configFN, "config", "", "Path to the config file")
 	flag.BoolVar(&jsonLog, "json-log", true, "produce JSON log output on verbose level")
 	flag.BoolVar(&verbose, "verbose", false, "Enable verbose logging")
@@ -113,32 +115,25 @@ func main() {
 		setupLog.Error(nil, "namespace cannot be empty")
 		os.Exit(1)
 	}
-
 	if cfg.Manager.SecretsNamespace == "" {
 		setupLog.Error(nil, "secretsNamespace cannot be empty")
 		os.Exit(1)
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                        scheme,
-		MetricsBindAddress:            cfg.Prometheus.Addr,
-		Port:                          9443,
-		HealthProbeBindAddress:        cfg.Health.Addr,
-		LeaderElection:                true,
-		LeaderElectionID:              "ws-manager-mk2-leader.gitpod.io",
-		LeaderElectionReleaseOnCancel: true,
-		NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
-			opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}
-			return cache.New(config, opts)
-		},
+		Scheme:                 scheme,
+		MetricsBindAddress:     cfg.Prometheus.Addr,
+		Port:                   9443,
+		HealthProbeBindAddress: cfg.Health.Addr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "ws-manager-mk2-leader.gitpod.io",
+		NewCache:               cache.MultiNamespacedCacheBuilder([]string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}),
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
 
-	mgrCtx := ctrl.SetupSignalHandler()
-
 	maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
 	if err != nil {
 		setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
@@ -153,21 +148,6 @@ func main() {
 	}
 
 	activity := activity.NewWorkspaceActivity()
-
-	go func() {
-		for {
-			select {
-			case <-mgrCtx.Done():
-				return
-			case <-mgr.Elected():
-				now := time.Now()
-				setupLog.Info("updating activity started time", "now", now)
-				activity.ManagerStartedAt = now
-				return
-			}
-		}
-	}()
-
 	timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
 	if err != nil {
 		setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
@@ -185,13 +165,11 @@ func main() {
 		setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
 		os.Exit(1)
 	}
-
 	if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
 		os.Exit(1)
 	}
-
-	if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil {
+	if err = maintenanceReconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
 		os.Exit(1)
 	}
@@ -213,7 +191,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(mgrCtx); err != nil {
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

components/ws-manager-mk2/service/manager.go

@@ -16,13 +16,25 @@ import (
 	validation "github.com/go-ozzo/ozzo-validation"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
 	"golang.org/x/xerrors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
+
+	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
+	"github.com/gitpod-io/gitpod/common-go/log"
+	"github.com/gitpod-io/gitpod/common-go/tracing"
+	"github.com/gitpod-io/gitpod/common-go/util"
+	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
+	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
+	wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
+	"github.com/gitpod-io/gitpod/ws-manager/api/config"
+	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
+
+	csapi "github.com/gitpod-io/gitpod/content-service/api"
+	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,17 +46,6 @@ import (
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-
-	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
-	"github.com/gitpod-io/gitpod/common-go/log"
-	"github.com/gitpod-io/gitpod/common-go/tracing"
-	"github.com/gitpod-io/gitpod/common-go/util"
-	csapi "github.com/gitpod-io/gitpod/content-service/api"
-	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
-	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
-	wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
-	"github.com/gitpod-io/gitpod/ws-manager/api/config"
-	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
 )
 
 const (

install/installer/pkg/components/ws-manager-mk2/deployment.go

@@ -5,18 +5,17 @@
 package wsmanagermk2
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
+	"github.com/gitpod-io/gitpod/installer/pkg/common"
+	wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
+	"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
-
-	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
-	"github.com/gitpod-io/gitpod/installer/pkg/common"
-	wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
-	"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
 )
 
 func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
@@ -59,6 +58,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 			Name: Component,
 			Args: []string{
 				"--config", "/config/config.json",
+				"--leader-elect",
 			},
 			Image:           ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerMk2.Version),
 			ImagePullPolicy: corev1.PullIfNotPresent,
@@ -176,7 +176,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 			},
 			Spec: appsv1.DeploymentSpec{
 				Selector: &metav1.LabelSelector{MatchLabels: labels},
-				Replicas: pointer.Int32(2),
+				Replicas: common.Replicas(ctx, Component),
 				Strategy: common.DeploymentStrategy,
 				Template: corev1.PodTemplateSpec{
 					ObjectMeta: metav1.ObjectMeta{

test/go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/prometheus/procfs v0.10.1
 	github.com/vishvananda/netns v0.0.4
 	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
-	golang.org/x/oauth2 v0.8.0
+	golang.org/x/oauth2 v0.6.0
 	golang.org/x/sync v0.2.0
 	golang.org/x/sys v0.11.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
@@ -32,7 +32,7 @@ require (
 	k8s.io/client-go v0.27.3
 	k8s.io/klog/v2 v2.90.1
 	k8s.io/kubectl v0.27.3
-	sigs.k8s.io/e2e-framework v0.2.0
+	sigs.k8s.io/e2e-framework v0.0.7
 )
 
 require (
@@ -44,7 +44,6 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
@@ -105,7 +104,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -155,8 +154,8 @@ require (
 	github.com/xlab/treeprint v1.1.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	go.uber.org/atomic v1.8.0 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
@@ -167,7 +166,7 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.57.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	honnef.co/go/tools v0.2.2 // indirect