Skip to content

Commit 1f4884b

Browse files
aledbfaledbf
committed
Enable leader election in wa-manager-mk2
1 parent e69acdf commit 1f4884b

File tree

12 files changed

+331
-63
lines changed

12 files changed

+331
-63
lines changed

‎components/ws-manager-mk2/cmd/sample-workspace/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@ import (
1010
"log"
1111
"time"
1212

13-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1413
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1514
"k8s.io/utils/pointer"
1615
"sigs.k8s.io/yaml"
16+
17+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1718
)
1819

1920
func main() {

‎components/ws-manager-mk2/config/manager/manager.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ spec:
3333
containers:
3434
- command:
3535
- /manager
36-
args:
37-
- --leader-elect
3836
image: controller:latest
3937
name: manager
4038
securityContext:

‎components/ws-manager-mk2/controllers/maintenance_controller.go

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"encoding/json"
1010
"fmt"
11+
"os"
1112
"sync"
1213
"time"
1314

@@ -17,7 +18,12 @@ import (
1718
"k8s.io/apimachinery/pkg/types"
1819
ctrl "sigs.k8s.io/controller-runtime"
1920
"sigs.k8s.io/controller-runtime/pkg/client"
21+
"sigs.k8s.io/controller-runtime/pkg/controller"
22+
"sigs.k8s.io/controller-runtime/pkg/event"
23+
"sigs.k8s.io/controller-runtime/pkg/handler"
2024
"sigs.k8s.io/controller-runtime/pkg/log"
25+
"sigs.k8s.io/controller-runtime/pkg/predicate"
26+
"sigs.k8s.io/controller-runtime/pkg/source"
2127
)
2228

2329
var (
@@ -106,9 +112,46 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti
106112
log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil)
107113
}
108114

109-
func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
110-
return ctrl.NewControllerManagedBy(mgr).
111-
Named("maintenance").
112-
For(&corev1.ConfigMap{}).
113-
Complete(r)
115+
func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
116+
// We need to use an unmanaged controller to avoid issues when the pod is in standby mode.
117+
// In that scenario, the controllers are not started and don't watch changes and only
118+
// observe the maintenance mode during the initialization.
119+
c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r})
120+
if err != nil {
121+
return err
122+
}
123+
124+
go func() {
125+
err = c.Start(ctx)
126+
if err != nil {
127+
log.FromContext(ctx).Error(err, "cannot start maintenance reconciler")
128+
os.Exit(1)
129+
}
130+
}()
131+
132+
return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{})
133+
}
134+
135+
type filterConfigMap struct {
136+
predicate.Funcs
137+
}
138+
139+
func (f filterConfigMap) Create(e event.CreateEvent) bool {
140+
return f.filter(e.Object)
141+
}
142+
143+
func (f filterConfigMap) Update(e event.UpdateEvent) bool {
144+
return f.filter(e.ObjectNew)
145+
}
146+
147+
func (f filterConfigMap) Generic(e event.GenericEvent) bool {
148+
return f.filter(e.Object)
149+
}
150+
151+
func (f filterConfigMap) filter(obj client.Object) bool {
152+
if obj == nil {
153+
return false
154+
}
155+
156+
return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace
114157
}

‎components/ws-manager-mk2/controllers/workspace_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -500,8 +500,8 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
500500
return []string{owner.Name}
501501
}
502502
err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, wsOwnerKey, idx)
503-
if err != nil {
504-
return err
503+
if err != nil && err.Error() == "informer has already started" {
504+
return nil
505505
}
506506

507507
return ctrl.NewControllerManagedBy(mgr).

‎components/ws-manager-mk2/main.go

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"google.golang.org/grpc/credentials"
1919
"google.golang.org/grpc/credentials/insecure"
2020
_ "k8s.io/client-go/plugin/pkg/client/auth"
21+
"k8s.io/client-go/rest"
2122

2223
"github.com/bombsimon/logrusr/v2"
2324
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
@@ -68,13 +69,9 @@ func init() {
6869
}
6970

7071
func main() {
71-
var enableLeaderElection bool
7272
var configFN string
7373
var jsonLog bool
7474
var verbose bool
75-
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
76-
"Enable leader election for controller manager. "+
77-
"Enabling this will ensure there is only one active controller manager.")
7875
flag.StringVar(&configFN, "config", "", "Path to the config file")
7976
flag.BoolVar(&jsonLog, "json-log", true, "produce JSON log output on verbose level")
8077
flag.BoolVar(&verbose, "verbose", false, "Enable verbose logging")
@@ -115,25 +112,32 @@ func main() {
115112
setupLog.Error(nil, "namespace cannot be empty")
116113
os.Exit(1)
117114
}
115+
118116
if cfg.Manager.SecretsNamespace == "" {
119117
setupLog.Error(nil, "secretsNamespace cannot be empty")
120118
os.Exit(1)
121119
}
122120

123121
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
124-
Scheme: scheme,
125-
MetricsBindAddress: cfg.Prometheus.Addr,
126-
Port: 9443,
127-
HealthProbeBindAddress: cfg.Health.Addr,
128-
LeaderElection: enableLeaderElection,
129-
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
130-
NewCache: cache.MultiNamespacedCacheBuilder([]string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}),
122+
Scheme: scheme,
123+
MetricsBindAddress: cfg.Prometheus.Addr,
124+
Port: 9443,
125+
HealthProbeBindAddress: cfg.Health.Addr,
126+
LeaderElection: true,
127+
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
128+
LeaderElectionReleaseOnCancel: true,
129+
NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
130+
opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}
131+
return cache.New(config, opts)
132+
},
131133
})
132134
if err != nil {
133135
setupLog.Error(err, "unable to start manager")
134136
os.Exit(1)
135137
}
136138

139+
mgrCtx := ctrl.SetupSignalHandler()
140+
137141
maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
138142
if err != nil {
139143
setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
@@ -148,6 +152,7 @@ func main() {
148152
}
149153

150154
activity := activity.NewWorkspaceActivity()
155+
151156
timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
152157
if err != nil {
153158
setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
@@ -165,11 +170,13 @@ func main() {
165170
setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
166171
os.Exit(1)
167172
}
173+
168174
if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
169175
setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
170176
os.Exit(1)
171177
}
172-
if err = maintenanceReconciler.SetupWithManager(mgr); err != nil {
178+
179+
if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil {
173180
setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
174181
os.Exit(1)
175182
}
@@ -191,10 +198,13 @@ func main() {
191198
}
192199

193200
setupLog.Info("starting manager")
194-
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
201+
if err := mgr.Start(mgrCtx); err != nil {
195202
setupLog.Error(err, "problem running manager")
196203
os.Exit(1)
197204
}
205+
206+
setupLog.Info("new leader elected")
207+
os.Exit(1)
198208
}
199209

200210
func setupGRPCService(cfg *config.ServiceConfiguration, k8s client.Client, activity *activity.WorkspaceActivity, maintenance maintenance.Maintenance) (*service.WorkspaceManagerServer, error) {

‎components/ws-manager-mk2/service/manager.go

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,13 @@ import (
1616
validation "github.com/go-ozzo/ozzo-validation"
1717
"github.com/opentracing/opentracing-go"
1818
"github.com/prometheus/client_golang/prometheus"
19+
"github.com/sirupsen/logrus"
1920
"golang.org/x/xerrors"
2021
"google.golang.org/grpc/codes"
2122
"google.golang.org/grpc/peer"
2223
"google.golang.org/grpc/status"
2324
"google.golang.org/protobuf/proto"
2425
"google.golang.org/protobuf/types/known/timestamppb"
25-
26-
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
27-
"github.com/gitpod-io/gitpod/common-go/log"
28-
"github.com/gitpod-io/gitpod/common-go/tracing"
29-
"github.com/gitpod-io/gitpod/common-go/util"
30-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
31-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
32-
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
33-
"github.com/gitpod-io/gitpod/ws-manager/api/config"
34-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
35-
36-
csapi "github.com/gitpod-io/gitpod/content-service/api"
37-
"github.com/sirupsen/logrus"
3826
corev1 "k8s.io/api/core/v1"
3927
"k8s.io/apimachinery/pkg/api/errors"
4028
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +34,17 @@ import (
4634
"k8s.io/utils/pointer"
4735
"sigs.k8s.io/controller-runtime/pkg/client"
4836
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
37+
38+
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
39+
"github.com/gitpod-io/gitpod/common-go/log"
40+
"github.com/gitpod-io/gitpod/common-go/tracing"
41+
"github.com/gitpod-io/gitpod/common-go/util"
42+
csapi "github.com/gitpod-io/gitpod/content-service/api"
43+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
44+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
45+
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
46+
"github.com/gitpod-io/gitpod/ws-manager/api/config"
47+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
4948
)
5049

5150
const (

‎install/installer/pkg/components/ws-manager-mk2/deployment.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,18 @@
55
package wsmanagermk2
66

77
import (
8-
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
9-
"github.com/gitpod-io/gitpod/installer/pkg/common"
10-
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
11-
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
128
appsv1 "k8s.io/api/apps/v1"
139
corev1 "k8s.io/api/core/v1"
1410
"k8s.io/apimachinery/pkg/api/resource"
1511
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1612
"k8s.io/apimachinery/pkg/runtime"
1713
"k8s.io/apimachinery/pkg/util/intstr"
1814
"k8s.io/utils/pointer"
15+
16+
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
17+
"github.com/gitpod-io/gitpod/installer/pkg/common"
18+
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
19+
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
1920
)
2021

2122
func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
@@ -58,7 +59,6 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
5859
Name: Component,
5960
Args: []string{
6061
"--config", "/config/config.json",
61-
"--leader-elect",
6262
},
6363
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerMk2.Version),
6464
ImagePullPolicy: corev1.PullIfNotPresent,
@@ -176,7 +176,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
176176
},
177177
Spec: appsv1.DeploymentSpec{
178178
Selector: &metav1.LabelSelector{MatchLabels: labels},
179-
Replicas: common.Replicas(ctx, Component),
179+
Replicas: pointer.Int32(2),
180180
Strategy: common.DeploymentStrategy,
181181
Template: corev1.PodTemplateSpec{
182182
ObjectMeta: metav1.ObjectMeta{

‎test/go.mod

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/prometheus/procfs v0.10.1
2121
github.com/vishvananda/netns v0.0.4
2222
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
23-
golang.org/x/oauth2 v0.6.0
23+
golang.org/x/oauth2 v0.8.0
2424
golang.org/x/sync v0.2.0
2525
golang.org/x/sys v0.11.0
2626
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
@@ -32,7 +32,7 @@ require (
3232
k8s.io/client-go v0.27.3
3333
k8s.io/klog/v2 v2.90.1
3434
k8s.io/kubectl v0.27.3
35-
sigs.k8s.io/e2e-framework v0.0.7
35+
sigs.k8s.io/e2e-framework v0.2.0
3636
)
3737

3838
require (
@@ -44,6 +44,7 @@ require (
4444
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
4545
github.com/BurntSushi/toml v0.4.1 // indirect
4646
github.com/MakeNowJust/heredoc v1.0.0 // indirect
47+
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
4748
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
4849
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect
4950
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
@@ -104,7 +105,7 @@ require (
104105
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
105106
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
106107
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
107-
github.com/hashicorp/golang-lru v0.5.4 // indirect
108+
github.com/hashicorp/golang-lru v0.5.1 // indirect
108109
github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect
109110
github.com/imdario/mergo v0.3.12 // indirect
110111
github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -154,8 +155,8 @@ require (
154155
github.com/xlab/treeprint v1.1.0 // indirect
155156
go.opencensus.io v0.24.0 // indirect
156157
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
157-
go.uber.org/atomic v1.8.0 // indirect
158-
golang.org/x/crypto v0.1.0 // indirect
158+
go.uber.org/atomic v1.7.0 // indirect
159+
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
159160
golang.org/x/mod v0.10.0 // indirect
160161
golang.org/x/net v0.10.0 // indirect
161162
golang.org/x/term v0.8.0 // indirect
@@ -166,7 +167,7 @@ require (
166167
google.golang.org/appengine v1.6.7 // indirect
167168
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
168169
gopkg.in/inf.v0 v0.9.1 // indirect
169-
gopkg.in/ini.v1 v1.62.0 // indirect
170+
gopkg.in/ini.v1 v1.57.0 // indirect
170171
gopkg.in/yaml.v2 v2.4.0 // indirect
171172
gopkg.in/yaml.v3 v3.0.1 // indirect
172173
honnef.co/go/tools v0.2.2 // indirect

0 commit comments

Comments
 (0)