Skip to content

Commit 7e3ccd1

Browse files
aledbfaledbf
authored
Revert "Enable leader election in ws-manager-mk2 (#18511)" (#18537)
This reverts commit 2838b78.
1 parent 2838b78 commit 7e3ccd1

File tree

11 files changed

+61
-358
lines changed

11 files changed

+61
-358
lines changed

‎components/ws-manager-mk2/cmd/sample-workspace/main.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,10 @@ import (
1010
"log"
1111
"time"
1212

13+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1314
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1415
"k8s.io/utils/pointer"
1516
"sigs.k8s.io/yaml"
16-
17-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1817
)
1918

2019
func main() {

‎components/ws-manager-mk2/config/manager/manager.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ spec:
3333
containers:
3434
- command:
3535
- /manager
36+
args:
37+
- --leader-elect
3638
image: controller:latest
3739
name: manager
3840
securityContext:

‎components/ws-manager-mk2/controllers/maintenance_controller.go

Lines changed: 5 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"context"
99
"encoding/json"
1010
"fmt"
11-
"os"
1211
"sync"
1312
"time"
1413

@@ -18,12 +17,7 @@ import (
1817
"k8s.io/apimachinery/pkg/types"
1918
ctrl "sigs.k8s.io/controller-runtime"
2019
"sigs.k8s.io/controller-runtime/pkg/client"
21-
"sigs.k8s.io/controller-runtime/pkg/controller"
22-
"sigs.k8s.io/controller-runtime/pkg/event"
23-
"sigs.k8s.io/controller-runtime/pkg/handler"
2420
"sigs.k8s.io/controller-runtime/pkg/log"
25-
"sigs.k8s.io/controller-runtime/pkg/predicate"
26-
"sigs.k8s.io/controller-runtime/pkg/source"
2721
)
2822

2923
var (
@@ -112,46 +106,9 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti
112106
log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil)
113107
}
114108

115-
func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
116-
// We need to use an unmanaged controller to avoid issues when the pod is in standby mode.
117-
// In that scenario, the controllers are not started and don't watch changes and only
118-
// observe the maintenance mode during the initialization.
119-
c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r})
120-
if err != nil {
121-
return err
122-
}
123-
124-
go func() {
125-
err = c.Start(ctx)
126-
if err != nil {
127-
log.FromContext(ctx).Error(err, "cannot start maintenance reconciler")
128-
os.Exit(1)
129-
}
130-
}()
131-
132-
return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{})
133-
}
134-
135-
type filterConfigMap struct {
136-
predicate.Funcs
137-
}
138-
139-
func (f filterConfigMap) Create(e event.CreateEvent) bool {
140-
return f.filter(e.Object)
141-
}
142-
143-
func (f filterConfigMap) Update(e event.UpdateEvent) bool {
144-
return f.filter(e.ObjectNew)
145-
}
146-
147-
func (f filterConfigMap) Generic(e event.GenericEvent) bool {
148-
return f.filter(e.Object)
149-
}
150-
151-
func (f filterConfigMap) filter(obj client.Object) bool {
152-
if obj == nil {
153-
return false
154-
}
155-
156-
return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace
109+
func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
110+
return ctrl.NewControllerManagedBy(mgr).
111+
Named("maintenance").
112+
For(&corev1.ConfigMap{}).
113+
Complete(r)
157114
}

‎components/ws-manager-mk2/main.go

Lines changed: 13 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,13 @@ import (
1111
"fmt"
1212
"net"
1313
"os"
14-
"time"
1514

1615
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
1716
// to ensure that exec-entrypoint and run can make use of them.
1817
"google.golang.org/grpc"
1918
"google.golang.org/grpc/credentials"
2019
"google.golang.org/grpc/credentials/insecure"
2120
_ "k8s.io/client-go/plugin/pkg/client/auth"
22-
"k8s.io/client-go/rest"
2321

2422
"github.com/bombsimon/logrusr/v2"
2523
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
@@ -70,9 +68,13 @@ func init() {
7068
}
7169

7270
func main() {
71+
var enableLeaderElection bool
7372
var configFN string
7473
var jsonLog bool
7574
var verbose bool
75+
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
76+
"Enable leader election for controller manager. "+
77+
"Enabling this will ensure there is only one active controller manager.")
7678
flag.StringVar(&configFN, "config", "", "Path to the config file")
7779
flag.BoolVar(&jsonLog, "json-log", true, "produce JSON log output on verbose level")
7880
flag.BoolVar(&verbose, "verbose", false, "Enable verbose logging")
@@ -113,32 +115,25 @@ func main() {
113115
setupLog.Error(nil, "namespace cannot be empty")
114116
os.Exit(1)
115117
}
116-
117118
if cfg.Manager.SecretsNamespace == "" {
118119
setupLog.Error(nil, "secretsNamespace cannot be empty")
119120
os.Exit(1)
120121
}
121122

122123
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
123-
Scheme: scheme,
124-
MetricsBindAddress: cfg.Prometheus.Addr,
125-
Port: 9443,
126-
HealthProbeBindAddress: cfg.Health.Addr,
127-
LeaderElection: true,
128-
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
129-
LeaderElectionReleaseOnCancel: true,
130-
NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
131-
opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}
132-
return cache.New(config, opts)
133-
},
124+
Scheme: scheme,
125+
MetricsBindAddress: cfg.Prometheus.Addr,
126+
Port: 9443,
127+
HealthProbeBindAddress: cfg.Health.Addr,
128+
LeaderElection: enableLeaderElection,
129+
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
130+
NewCache: cache.MultiNamespacedCacheBuilder([]string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}),
134131
})
135132
if err != nil {
136133
setupLog.Error(err, "unable to start manager")
137134
os.Exit(1)
138135
}
139136

140-
mgrCtx := ctrl.SetupSignalHandler()
141-
142137
maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
143138
if err != nil {
144139
setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
@@ -153,21 +148,6 @@ func main() {
153148
}
154149

155150
activity := activity.NewWorkspaceActivity()
156-
157-
go func() {
158-
for {
159-
select {
160-
case <-mgrCtx.Done():
161-
return
162-
case <-mgr.Elected():
163-
now := time.Now()
164-
setupLog.Info("updating activity started time", "now", now)
165-
activity.ManagerStartedAt = now
166-
return
167-
}
168-
}
169-
}()
170-
171151
timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
172152
if err != nil {
173153
setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
@@ -185,13 +165,11 @@ func main() {
185165
setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
186166
os.Exit(1)
187167
}
188-
189168
if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
190169
setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
191170
os.Exit(1)
192171
}
193-
194-
if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil {
172+
if err = maintenanceReconciler.SetupWithManager(mgr); err != nil {
195173
setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
196174
os.Exit(1)
197175
}
@@ -213,7 +191,7 @@ func main() {
213191
}
214192

215193
setupLog.Info("starting manager")
216-
if err := mgr.Start(mgrCtx); err != nil {
194+
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
217195
setupLog.Error(err, "problem running manager")
218196
os.Exit(1)
219197
}

‎components/ws-manager-mk2/service/manager.go

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,25 @@ import (
1616
validation "github.com/go-ozzo/ozzo-validation"
1717
"github.com/opentracing/opentracing-go"
1818
"github.com/prometheus/client_golang/prometheus"
19-
"github.com/sirupsen/logrus"
2019
"golang.org/x/xerrors"
2120
"google.golang.org/grpc/codes"
2221
"google.golang.org/grpc/peer"
2322
"google.golang.org/grpc/status"
2423
"google.golang.org/protobuf/proto"
2524
"google.golang.org/protobuf/types/known/timestamppb"
25+
26+
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
27+
"github.com/gitpod-io/gitpod/common-go/log"
28+
"github.com/gitpod-io/gitpod/common-go/tracing"
29+
"github.com/gitpod-io/gitpod/common-go/util"
30+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
31+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
32+
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
33+
"github.com/gitpod-io/gitpod/ws-manager/api/config"
34+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
35+
36+
csapi "github.com/gitpod-io/gitpod/content-service/api"
37+
"github.com/sirupsen/logrus"
2638
corev1 "k8s.io/api/core/v1"
2739
"k8s.io/apimachinery/pkg/api/errors"
2840
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,17 +46,6 @@ import (
3446
"k8s.io/utils/pointer"
3547
"sigs.k8s.io/controller-runtime/pkg/client"
3648
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
37-
38-
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
39-
"github.com/gitpod-io/gitpod/common-go/log"
40-
"github.com/gitpod-io/gitpod/common-go/tracing"
41-
"github.com/gitpod-io/gitpod/common-go/util"
42-
csapi "github.com/gitpod-io/gitpod/content-service/api"
43-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
44-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
45-
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
46-
"github.com/gitpod-io/gitpod/ws-manager/api/config"
47-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
4849
)
4950

5051
const (

‎install/installer/pkg/components/ws-manager-mk2/deployment.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,17 @@
55
package wsmanagermk2
66

77
import (
8+
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
9+
"github.com/gitpod-io/gitpod/installer/pkg/common"
10+
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
11+
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
812
appsv1 "k8s.io/api/apps/v1"
913
corev1 "k8s.io/api/core/v1"
1014
"k8s.io/apimachinery/pkg/api/resource"
1115
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1216
"k8s.io/apimachinery/pkg/runtime"
1317
"k8s.io/apimachinery/pkg/util/intstr"
1418
"k8s.io/utils/pointer"
15-
16-
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
17-
"github.com/gitpod-io/gitpod/installer/pkg/common"
18-
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
19-
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
2019
)
2120

2221
func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
@@ -59,6 +58,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
5958
Name: Component,
6059
Args: []string{
6160
"--config", "/config/config.json",
61+
"--leader-elect",
6262
},
6363
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerMk2.Version),
6464
ImagePullPolicy: corev1.PullIfNotPresent,
@@ -176,7 +176,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
176176
},
177177
Spec: appsv1.DeploymentSpec{
178178
Selector: &metav1.LabelSelector{MatchLabels: labels},
179-
Replicas: pointer.Int32(2),
179+
Replicas: common.Replicas(ctx, Component),
180180
Strategy: common.DeploymentStrategy,
181181
Template: corev1.PodTemplateSpec{
182182
ObjectMeta: metav1.ObjectMeta{

‎test/go.mod

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/prometheus/procfs v0.10.1
2121
github.com/vishvananda/netns v0.0.4
2222
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
23-
golang.org/x/oauth2 v0.8.0
23+
golang.org/x/oauth2 v0.6.0
2424
golang.org/x/sync v0.2.0
2525
golang.org/x/sys v0.11.0
2626
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
@@ -32,7 +32,7 @@ require (
3232
k8s.io/client-go v0.27.3
3333
k8s.io/klog/v2 v2.90.1
3434
k8s.io/kubectl v0.27.3
35-
sigs.k8s.io/e2e-framework v0.2.0
35+
sigs.k8s.io/e2e-framework v0.0.7
3636
)
3737

3838
require (
@@ -44,7 +44,6 @@ require (
4444
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
4545
github.com/BurntSushi/toml v0.4.1 // indirect
4646
github.com/MakeNowJust/heredoc v1.0.0 // indirect
47-
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
4847
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
4948
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect
5049
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
@@ -105,7 +104,7 @@ require (
105104
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
106105
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
107106
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
108-
github.com/hashicorp/golang-lru v0.5.1 // indirect
107+
github.com/hashicorp/golang-lru v0.5.4 // indirect
109108
github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect
110109
github.com/imdario/mergo v0.3.12 // indirect
111110
github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -155,8 +154,8 @@ require (
155154
github.com/xlab/treeprint v1.1.0 // indirect
156155
go.opencensus.io v0.24.0 // indirect
157156
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
158-
go.uber.org/atomic v1.7.0 // indirect
159-
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
157+
go.uber.org/atomic v1.8.0 // indirect
158+
golang.org/x/crypto v0.1.0 // indirect
160159
golang.org/x/mod v0.10.0 // indirect
161160
golang.org/x/net v0.10.0 // indirect
162161
golang.org/x/term v0.8.0 // indirect
@@ -167,7 +166,7 @@ require (
167166
google.golang.org/appengine v1.6.7 // indirect
168167
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
169168
gopkg.in/inf.v0 v0.9.1 // indirect
170-
gopkg.in/ini.v1 v1.57.0 // indirect
169+
gopkg.in/ini.v1 v1.62.0 // indirect
171170
gopkg.in/yaml.v2 v2.4.0 // indirect
172171
gopkg.in/yaml.v3 v3.0.1 // indirect
173172
honnef.co/go/tools v0.2.2 // indirect

0 commit comments

Comments
 (0)