Skip to content

Commit 2838b78

Browse files
aledbfaledbf
authored
Enable leader election in ws-manager-mk2 (#18511)
* Enable leader election in wa-manager-mk2 * Switch to four parallel tests * Cleanup * Update activity started time after leader election * Count maintenance test failures * Wait before updating the activity started time
1 parent 553f9b2 commit 2838b78

File tree

11 files changed

+358
-61
lines changed

11 files changed

+358
-61
lines changed

‎components/ws-manager-mk2/cmd/sample-workspace/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@ import (
1010
"log"
1111
"time"
1212

13-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1413
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1514
"k8s.io/utils/pointer"
1615
"sigs.k8s.io/yaml"
16+
17+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1718
)
1819

1920
func main() {

‎components/ws-manager-mk2/config/manager/manager.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ spec:
3333
containers:
3434
- command:
3535
- /manager
36-
args:
37-
- --leader-elect
3836
image: controller:latest
3937
name: manager
4038
securityContext:

‎components/ws-manager-mk2/controllers/maintenance_controller.go

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"encoding/json"
1010
"fmt"
11+
"os"
1112
"sync"
1213
"time"
1314

@@ -17,7 +18,12 @@ import (
1718
"k8s.io/apimachinery/pkg/types"
1819
ctrl "sigs.k8s.io/controller-runtime"
1920
"sigs.k8s.io/controller-runtime/pkg/client"
21+
"sigs.k8s.io/controller-runtime/pkg/controller"
22+
"sigs.k8s.io/controller-runtime/pkg/event"
23+
"sigs.k8s.io/controller-runtime/pkg/handler"
2024
"sigs.k8s.io/controller-runtime/pkg/log"
25+
"sigs.k8s.io/controller-runtime/pkg/predicate"
26+
"sigs.k8s.io/controller-runtime/pkg/source"
2127
)
2228

2329
var (
@@ -106,9 +112,46 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti
106112
log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil)
107113
}
108114

109-
func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
110-
return ctrl.NewControllerManagedBy(mgr).
111-
Named("maintenance").
112-
For(&corev1.ConfigMap{}).
113-
Complete(r)
115+
func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
116+
// We need to use an unmanaged controller to avoid issues when the pod is in standby mode.
117+
// In that scenario, the controllers are not started and don't watch changes and only
118+
// observe the maintenance mode during the initialization.
119+
c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r})
120+
if err != nil {
121+
return err
122+
}
123+
124+
go func() {
125+
err = c.Start(ctx)
126+
if err != nil {
127+
log.FromContext(ctx).Error(err, "cannot start maintenance reconciler")
128+
os.Exit(1)
129+
}
130+
}()
131+
132+
return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{})
133+
}
134+
135+
type filterConfigMap struct {
136+
predicate.Funcs
137+
}
138+
139+
func (f filterConfigMap) Create(e event.CreateEvent) bool {
140+
return f.filter(e.Object)
141+
}
142+
143+
func (f filterConfigMap) Update(e event.UpdateEvent) bool {
144+
return f.filter(e.ObjectNew)
145+
}
146+
147+
func (f filterConfigMap) Generic(e event.GenericEvent) bool {
148+
return f.filter(e.Object)
149+
}
150+
151+
func (f filterConfigMap) filter(obj client.Object) bool {
152+
if obj == nil {
153+
return false
154+
}
155+
156+
return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace
114157
}

‎components/ws-manager-mk2/main.go

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ import (
1111
"fmt"
1212
"net"
1313
"os"
14+
"time"
1415

1516
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
1617
// to ensure that exec-entrypoint and run can make use of them.
1718
"google.golang.org/grpc"
1819
"google.golang.org/grpc/credentials"
1920
"google.golang.org/grpc/credentials/insecure"
2021
_ "k8s.io/client-go/plugin/pkg/client/auth"
22+
"k8s.io/client-go/rest"
2123

2224
"github.com/bombsimon/logrusr/v2"
2325
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
@@ -68,13 +70,9 @@ func init() {
6870
}
6971

7072
func main() {
71-
var enableLeaderElection bool
7273
var configFN string
7374
var jsonLog bool
7475
var verbose bool
75-
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
76-
"Enable leader election for controller manager. "+
77-
"Enabling this will ensure there is only one active controller manager.")
7876
flag.StringVar(&configFN, "config", "", "Path to the config file")
7977
flag.BoolVar(&jsonLog, "json-log", true, "produce JSON log output on verbose level")
8078
flag.BoolVar(&verbose, "verbose", false, "Enable verbose logging")
@@ -115,25 +113,32 @@ func main() {
115113
setupLog.Error(nil, "namespace cannot be empty")
116114
os.Exit(1)
117115
}
116+
118117
if cfg.Manager.SecretsNamespace == "" {
119118
setupLog.Error(nil, "secretsNamespace cannot be empty")
120119
os.Exit(1)
121120
}
122121

123122
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
124-
Scheme: scheme,
125-
MetricsBindAddress: cfg.Prometheus.Addr,
126-
Port: 9443,
127-
HealthProbeBindAddress: cfg.Health.Addr,
128-
LeaderElection: enableLeaderElection,
129-
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
130-
NewCache: cache.MultiNamespacedCacheBuilder([]string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}),
123+
Scheme: scheme,
124+
MetricsBindAddress: cfg.Prometheus.Addr,
125+
Port: 9443,
126+
HealthProbeBindAddress: cfg.Health.Addr,
127+
LeaderElection: true,
128+
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
129+
LeaderElectionReleaseOnCancel: true,
130+
NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
131+
opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}
132+
return cache.New(config, opts)
133+
},
131134
})
132135
if err != nil {
133136
setupLog.Error(err, "unable to start manager")
134137
os.Exit(1)
135138
}
136139

140+
mgrCtx := ctrl.SetupSignalHandler()
141+
137142
maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
138143
if err != nil {
139144
setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
@@ -148,6 +153,21 @@ func main() {
148153
}
149154

150155
activity := activity.NewWorkspaceActivity()
156+
157+
go func() {
158+
for {
159+
select {
160+
case <-mgrCtx.Done():
161+
return
162+
case <-mgr.Elected():
163+
now := time.Now()
164+
setupLog.Info("updating activity started time", "now", now)
165+
activity.ManagerStartedAt = now
166+
return
167+
}
168+
}
169+
}()
170+
151171
timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
152172
if err != nil {
153173
setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
@@ -165,11 +185,13 @@ func main() {
165185
setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
166186
os.Exit(1)
167187
}
188+
168189
if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
169190
setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
170191
os.Exit(1)
171192
}
172-
if err = maintenanceReconciler.SetupWithManager(mgr); err != nil {
193+
194+
if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil {
173195
setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
174196
os.Exit(1)
175197
}
@@ -191,7 +213,7 @@ func main() {
191213
}
192214

193215
setupLog.Info("starting manager")
194-
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
216+
if err := mgr.Start(mgrCtx); err != nil {
195217
setupLog.Error(err, "problem running manager")
196218
os.Exit(1)
197219
}

‎components/ws-manager-mk2/service/manager.go

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,13 @@ import (
1616
validation "github.com/go-ozzo/ozzo-validation"
1717
"github.com/opentracing/opentracing-go"
1818
"github.com/prometheus/client_golang/prometheus"
19+
"github.com/sirupsen/logrus"
1920
"golang.org/x/xerrors"
2021
"google.golang.org/grpc/codes"
2122
"google.golang.org/grpc/peer"
2223
"google.golang.org/grpc/status"
2324
"google.golang.org/protobuf/proto"
2425
"google.golang.org/protobuf/types/known/timestamppb"
25-
26-
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
27-
"github.com/gitpod-io/gitpod/common-go/log"
28-
"github.com/gitpod-io/gitpod/common-go/tracing"
29-
"github.com/gitpod-io/gitpod/common-go/util"
30-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
31-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
32-
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
33-
"github.com/gitpod-io/gitpod/ws-manager/api/config"
34-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
35-
36-
csapi "github.com/gitpod-io/gitpod/content-service/api"
37-
"github.com/sirupsen/logrus"
3826
corev1 "k8s.io/api/core/v1"
3927
"k8s.io/apimachinery/pkg/api/errors"
4028
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +34,17 @@ import (
4634
"k8s.io/utils/pointer"
4735
"sigs.k8s.io/controller-runtime/pkg/client"
4836
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
37+
38+
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
39+
"github.com/gitpod-io/gitpod/common-go/log"
40+
"github.com/gitpod-io/gitpod/common-go/tracing"
41+
"github.com/gitpod-io/gitpod/common-go/util"
42+
csapi "github.com/gitpod-io/gitpod/content-service/api"
43+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
44+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
45+
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
46+
"github.com/gitpod-io/gitpod/ws-manager/api/config"
47+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
4948
)
5049

5150
const (

‎install/installer/pkg/components/ws-manager-mk2/deployment.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,18 @@
55
package wsmanagermk2
66

77
import (
8-
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
9-
"github.com/gitpod-io/gitpod/installer/pkg/common"
10-
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
11-
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
128
appsv1 "k8s.io/api/apps/v1"
139
corev1 "k8s.io/api/core/v1"
1410
"k8s.io/apimachinery/pkg/api/resource"
1511
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1612
"k8s.io/apimachinery/pkg/runtime"
1713
"k8s.io/apimachinery/pkg/util/intstr"
1814
"k8s.io/utils/pointer"
15+
16+
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
17+
"github.com/gitpod-io/gitpod/installer/pkg/common"
18+
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
19+
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
1920
)
2021

2122
func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
@@ -58,7 +59,6 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
5859
Name: Component,
5960
Args: []string{
6061
"--config", "/config/config.json",
61-
"--leader-elect",
6262
},
6363
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerMk2.Version),
6464
ImagePullPolicy: corev1.PullIfNotPresent,
@@ -176,7 +176,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
176176
},
177177
Spec: appsv1.DeploymentSpec{
178178
Selector: &metav1.LabelSelector{MatchLabels: labels},
179-
Replicas: common.Replicas(ctx, Component),
179+
Replicas: pointer.Int32(2),
180180
Strategy: common.DeploymentStrategy,
181181
Template: corev1.PodTemplateSpec{
182182
ObjectMeta: metav1.ObjectMeta{

‎test/go.mod

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/prometheus/procfs v0.10.1
2121
github.com/vishvananda/netns v0.0.4
2222
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
23-
golang.org/x/oauth2 v0.6.0
23+
golang.org/x/oauth2 v0.8.0
2424
golang.org/x/sync v0.2.0
2525
golang.org/x/sys v0.11.0
2626
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
@@ -32,7 +32,7 @@ require (
3232
k8s.io/client-go v0.27.3
3333
k8s.io/klog/v2 v2.90.1
3434
k8s.io/kubectl v0.27.3
35-
sigs.k8s.io/e2e-framework v0.0.7
35+
sigs.k8s.io/e2e-framework v0.2.0
3636
)
3737

3838
require (
@@ -44,6 +44,7 @@ require (
4444
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
4545
github.com/BurntSushi/toml v0.4.1 // indirect
4646
github.com/MakeNowJust/heredoc v1.0.0 // indirect
47+
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
4748
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
4849
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect
4950
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
@@ -104,7 +105,7 @@ require (
104105
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
105106
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
106107
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
107-
github.com/hashicorp/golang-lru v0.5.4 // indirect
108+
github.com/hashicorp/golang-lru v0.5.1 // indirect
108109
github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect
109110
github.com/imdario/mergo v0.3.12 // indirect
110111
github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -154,8 +155,8 @@ require (
154155
github.com/xlab/treeprint v1.1.0 // indirect
155156
go.opencensus.io v0.24.0 // indirect
156157
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
157-
go.uber.org/atomic v1.8.0 // indirect
158-
golang.org/x/crypto v0.1.0 // indirect
158+
go.uber.org/atomic v1.7.0 // indirect
159+
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
159160
golang.org/x/mod v0.10.0 // indirect
160161
golang.org/x/net v0.10.0 // indirect
161162
golang.org/x/term v0.8.0 // indirect
@@ -166,7 +167,7 @@ require (
166167
google.golang.org/appengine v1.6.7 // indirect
167168
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
168169
gopkg.in/inf.v0 v0.9.1 // indirect
169-
gopkg.in/ini.v1 v1.62.0 // indirect
170+
gopkg.in/ini.v1 v1.57.0 // indirect
170171
gopkg.in/yaml.v2 v2.4.0 // indirect
171172
gopkg.in/yaml.v3 v3.0.1 // indirect
172173
honnef.co/go/tools v0.2.2 // indirect

0 commit comments

Comments
 (0)