Skip to content

Commit f7912a0

Browse files
aledbfaledbf
committed
Enable leader election in wa-manager-mk2
1 parent e69acdf commit f7912a0

File tree

9 files changed

+226
-43
lines changed

9 files changed

+226
-43
lines changed

‎components/ws-manager-mk2/cmd/sample-workspace/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@ import (
1010
"log"
1111
"time"
1212

13-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1413
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1514
"k8s.io/utils/pointer"
1615
"sigs.k8s.io/yaml"
16+
17+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1718
)
1819

1920
func main() {

‎components/ws-manager-mk2/config/manager/manager.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ spec:
3333
containers:
3434
- command:
3535
- /manager
36-
args:
37-
- --leader-elect
3836
image: controller:latest
3937
name: manager
4038
securityContext:

‎components/ws-manager-mk2/controllers/maintenance_controller.go

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"encoding/json"
1010
"fmt"
11+
"os"
1112
"sync"
1213
"time"
1314

@@ -17,7 +18,12 @@ import (
1718
"k8s.io/apimachinery/pkg/types"
1819
ctrl "sigs.k8s.io/controller-runtime"
1920
"sigs.k8s.io/controller-runtime/pkg/client"
21+
"sigs.k8s.io/controller-runtime/pkg/controller"
22+
"sigs.k8s.io/controller-runtime/pkg/event"
23+
"sigs.k8s.io/controller-runtime/pkg/handler"
2024
"sigs.k8s.io/controller-runtime/pkg/log"
25+
"sigs.k8s.io/controller-runtime/pkg/predicate"
26+
"sigs.k8s.io/controller-runtime/pkg/source"
2127
)
2228

2329
var (
@@ -106,9 +112,46 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti
106112
log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil)
107113
}
108114

109-
func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
110-
return ctrl.NewControllerManagedBy(mgr).
111-
Named("maintenance").
112-
For(&corev1.ConfigMap{}).
113-
Complete(r)
115+
func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
116+
// We need to use an unmanaged controller to avoid issues when the pod is in standby mode.
117+
// In that scenario, the controllers are not started and don't watch changes and only
118+
// observe the maintenance mode during the initialization.
119+
c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r})
120+
if err != nil {
121+
return err
122+
}
123+
124+
go func() {
125+
err = c.Start(ctx)
126+
if err != nil {
127+
log.FromContext(ctx).Error(err, "cannot start maintenance reconciler")
128+
os.Exit(1)
129+
}
130+
}()
131+
132+
return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{})
133+
}
134+
135+
type filterConfigMap struct {
136+
predicate.Funcs
137+
}
138+
139+
func (f filterConfigMap) Create(e event.CreateEvent) bool {
140+
return f.filter(e.Object)
141+
}
142+
143+
func (f filterConfigMap) Update(e event.UpdateEvent) bool {
144+
return f.filter(e.ObjectNew)
145+
}
146+
147+
func (f filterConfigMap) Generic(e event.GenericEvent) bool {
148+
return f.filter(e.Object)
149+
}
150+
151+
func (f filterConfigMap) filter(obj client.Object) bool {
152+
if obj == nil {
153+
return false
154+
}
155+
156+
return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace
114157
}

‎components/ws-manager-mk2/controllers/workspace_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -500,8 +500,8 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
500500
return []string{owner.Name}
501501
}
502502
err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, wsOwnerKey, idx)
503-
if err != nil {
504-
return err
503+
if err != nil && err.Error() == "informer has already started" {
504+
return nil
505505
}
506506

507507
return ctrl.NewControllerManagedBy(mgr).

‎components/ws-manager-mk2/main.go

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"google.golang.org/grpc/credentials"
1919
"google.golang.org/grpc/credentials/insecure"
2020
_ "k8s.io/client-go/plugin/pkg/client/auth"
21+
"k8s.io/client-go/rest"
2122

2223
"github.com/bombsimon/logrusr/v2"
2324
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
@@ -68,13 +69,9 @@ func init() {
6869
}
6970

7071
func main() {
71-
var enableLeaderElection bool
7272
var configFN string
7373
var jsonLog bool
7474
var verbose bool
75-
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
76-
"Enable leader election for controller manager. "+
77-
"Enabling this will ensure there is only one active controller manager.")
7875
flag.StringVar(&configFN, "config", "", "Path to the config file")
7976
flag.BoolVar(&jsonLog, "json-log", true, "produce JSON log output on verbose level")
8077
flag.BoolVar(&verbose, "verbose", false, "Enable verbose logging")
@@ -115,25 +112,32 @@ func main() {
115112
setupLog.Error(nil, "namespace cannot be empty")
116113
os.Exit(1)
117114
}
115+
118116
if cfg.Manager.SecretsNamespace == "" {
119117
setupLog.Error(nil, "secretsNamespace cannot be empty")
120118
os.Exit(1)
121119
}
122120

123121
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
124-
Scheme: scheme,
125-
MetricsBindAddress: cfg.Prometheus.Addr,
126-
Port: 9443,
127-
HealthProbeBindAddress: cfg.Health.Addr,
128-
LeaderElection: enableLeaderElection,
129-
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
130-
NewCache: cache.MultiNamespacedCacheBuilder([]string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}),
122+
Scheme: scheme,
123+
MetricsBindAddress: cfg.Prometheus.Addr,
124+
Port: 9443,
125+
HealthProbeBindAddress: cfg.Health.Addr,
126+
LeaderElection: true,
127+
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
128+
LeaderElectionReleaseOnCancel: true,
129+
NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
130+
opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}
131+
return cache.New(config, opts)
132+
},
131133
})
132134
if err != nil {
133135
setupLog.Error(err, "unable to start manager")
134136
os.Exit(1)
135137
}
136138

139+
mgrCtx := ctrl.SetupSignalHandler()
140+
137141
maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
138142
if err != nil {
139143
setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
@@ -148,6 +152,7 @@ func main() {
148152
}
149153

150154
activity := activity.NewWorkspaceActivity()
155+
151156
timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
152157
if err != nil {
153158
setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
@@ -165,11 +170,13 @@ func main() {
165170
setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
166171
os.Exit(1)
167172
}
173+
168174
if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
169175
setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
170176
os.Exit(1)
171177
}
172-
if err = maintenanceReconciler.SetupWithManager(mgr); err != nil {
178+
179+
if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil {
173180
setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
174181
os.Exit(1)
175182
}
@@ -191,10 +198,13 @@ func main() {
191198
}
192199

193200
setupLog.Info("starting manager")
194-
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
201+
if err := mgr.Start(mgrCtx); err != nil {
195202
setupLog.Error(err, "problem running manager")
196203
os.Exit(1)
197204
}
205+
206+
setupLog.Info("new leader elected")
207+
os.Exit(1)
198208
}
199209

200210
func setupGRPCService(cfg *config.ServiceConfiguration, k8s client.Client, activity *activity.WorkspaceActivity, maintenance maintenance.Maintenance) (*service.WorkspaceManagerServer, error) {

‎components/ws-manager-mk2/service/manager.go

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,13 @@ import (
1616
validation "github.com/go-ozzo/ozzo-validation"
1717
"github.com/opentracing/opentracing-go"
1818
"github.com/prometheus/client_golang/prometheus"
19+
"github.com/sirupsen/logrus"
1920
"golang.org/x/xerrors"
2021
"google.golang.org/grpc/codes"
2122
"google.golang.org/grpc/peer"
2223
"google.golang.org/grpc/status"
2324
"google.golang.org/protobuf/proto"
2425
"google.golang.org/protobuf/types/known/timestamppb"
25-
26-
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
27-
"github.com/gitpod-io/gitpod/common-go/log"
28-
"github.com/gitpod-io/gitpod/common-go/tracing"
29-
"github.com/gitpod-io/gitpod/common-go/util"
30-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
31-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
32-
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
33-
"github.com/gitpod-io/gitpod/ws-manager/api/config"
34-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
35-
36-
csapi "github.com/gitpod-io/gitpod/content-service/api"
37-
"github.com/sirupsen/logrus"
3826
corev1 "k8s.io/api/core/v1"
3927
"k8s.io/apimachinery/pkg/api/errors"
4028
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +34,17 @@ import (
4634
"k8s.io/utils/pointer"
4735
"sigs.k8s.io/controller-runtime/pkg/client"
4836
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
37+
38+
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
39+
"github.com/gitpod-io/gitpod/common-go/log"
40+
"github.com/gitpod-io/gitpod/common-go/tracing"
41+
"github.com/gitpod-io/gitpod/common-go/util"
42+
csapi "github.com/gitpod-io/gitpod/content-service/api"
43+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
44+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
45+
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
46+
"github.com/gitpod-io/gitpod/ws-manager/api/config"
47+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
4948
)
5049

5150
const (

‎install/installer/pkg/components/ws-manager-mk2/deployment.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,18 @@
55
package wsmanagermk2
66

77
import (
8-
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
9-
"github.com/gitpod-io/gitpod/installer/pkg/common"
10-
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
11-
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
128
appsv1 "k8s.io/api/apps/v1"
139
corev1 "k8s.io/api/core/v1"
1410
"k8s.io/apimachinery/pkg/api/resource"
1511
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1612
"k8s.io/apimachinery/pkg/runtime"
1713
"k8s.io/apimachinery/pkg/util/intstr"
1814
"k8s.io/utils/pointer"
15+
16+
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
17+
"github.com/gitpod-io/gitpod/installer/pkg/common"
18+
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
19+
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
1920
)
2021

2122
func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
@@ -58,7 +59,6 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
5859
Name: Component,
5960
Args: []string{
6061
"--config", "/config/config.json",
61-
"--leader-elect",
6262
},
6363
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerMk2.Version),
6464
ImagePullPolicy: corev1.PullIfNotPresent,
@@ -176,7 +176,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
176176
},
177177
Spec: appsv1.DeploymentSpec{
178178
Selector: &metav1.LabelSelector{MatchLabels: labels},
179-
Replicas: common.Replicas(ctx, Component),
179+
Replicas: pointer.Int32(2),
180180
Strategy: common.DeploymentStrategy,
181181
Template: corev1.PodTemplateSpec{
182182
ObjectMeta: metav1.ObjectMeta{

‎test/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ if [ "$TEST_SUITE" == "workspace" ]; then
114114

115115
set +e
116116
# shellcheck disable=SC2086
117-
go test -p 10 -v $TEST_LIST "${args[@]}" -parallel-features=true 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy
117+
go test -p 6 -v $TEST_LIST "${args[@]}" -parallel-features=true 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy
118118
RC=${PIPESTATUS[0]}
119119
set -e
120120

0 commit comments

Comments
 (0)