Skip to content

Commit 569d1b8

Browse files
aledbfaledbf
committed
Enable leader election in ws-manager-mk2
1 parent 7e3ccd1 commit 569d1b8

File tree

9 files changed

+337
-42
lines changed

9 files changed

+337
-42
lines changed

‎components/ws-manager-mk2/cmd/sample-workspace/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@ import (
1010
"log"
1111
"time"
1212

13-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1413
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1514
"k8s.io/utils/pointer"
1615
"sigs.k8s.io/yaml"
16+
17+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
1718
)
1819

1920
func main() {

‎components/ws-manager-mk2/config/manager/manager.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ spec:
3333
containers:
3434
- command:
3535
- /manager
36-
args:
37-
- --leader-elect
3836
image: controller:latest
3937
name: manager
4038
securityContext:

‎components/ws-manager-mk2/controllers/maintenance_controller.go

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"encoding/json"
1010
"fmt"
11+
"os"
1112
"sync"
1213
"time"
1314

@@ -17,7 +18,12 @@ import (
1718
"k8s.io/apimachinery/pkg/types"
1819
ctrl "sigs.k8s.io/controller-runtime"
1920
"sigs.k8s.io/controller-runtime/pkg/client"
21+
"sigs.k8s.io/controller-runtime/pkg/controller"
22+
"sigs.k8s.io/controller-runtime/pkg/event"
23+
"sigs.k8s.io/controller-runtime/pkg/handler"
2024
"sigs.k8s.io/controller-runtime/pkg/log"
25+
"sigs.k8s.io/controller-runtime/pkg/predicate"
26+
"sigs.k8s.io/controller-runtime/pkg/source"
2127
)
2228

2329
var (
@@ -106,9 +112,46 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti
106112
log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil)
107113
}
108114

109-
func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
110-
return ctrl.NewControllerManagedBy(mgr).
111-
Named("maintenance").
112-
For(&corev1.ConfigMap{}).
113-
Complete(r)
115+
func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
116+
// We need to use an unmanaged controller to avoid issues when the pod is in standby mode.
117+
// In that scenario, the controllers are not started and don't watch changes and only
118+
// observe the maintenance mode during the initialization.
119+
c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r})
120+
if err != nil {
121+
return err
122+
}
123+
124+
go func() {
125+
err = c.Start(ctx)
126+
if err != nil {
127+
log.FromContext(ctx).Error(err, "cannot start maintenance reconciler")
128+
os.Exit(1)
129+
}
130+
}()
131+
132+
return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{})
133+
}
134+
135+
type filterConfigMap struct {
136+
predicate.Funcs
137+
}
138+
139+
func (f filterConfigMap) Create(e event.CreateEvent) bool {
140+
return f.filter(e.Object)
141+
}
142+
143+
func (f filterConfigMap) Update(e event.UpdateEvent) bool {
144+
return f.filter(e.ObjectNew)
145+
}
146+
147+
func (f filterConfigMap) Generic(e event.GenericEvent) bool {
148+
return f.filter(e.Object)
149+
}
150+
151+
func (f filterConfigMap) filter(obj client.Object) bool {
152+
if obj == nil {
153+
return false
154+
}
155+
156+
return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace
114157
}

‎components/ws-manager-mk2/main.go

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ import (
1111
"fmt"
1212
"net"
1313
"os"
14+
"time"
1415

1516
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
1617
// to ensure that exec-entrypoint and run can make use of them.
1718
"google.golang.org/grpc"
1819
"google.golang.org/grpc/credentials"
1920
"google.golang.org/grpc/credentials/insecure"
2021
_ "k8s.io/client-go/plugin/pkg/client/auth"
22+
"k8s.io/client-go/rest"
2123

2224
"github.com/bombsimon/logrusr/v2"
2325
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
@@ -68,13 +70,9 @@ func init() {
6870
}
6971

7072
func main() {
71-
var enableLeaderElection bool
7273
var configFN string
7374
var jsonLog bool
7475
var verbose bool
75-
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
76-
"Enable leader election for controller manager. "+
77-
"Enabling this will ensure there is only one active controller manager.")
7876
flag.StringVar(&configFN, "config", "", "Path to the config file")
7977
flag.BoolVar(&jsonLog, "json-log", true, "produce JSON log output on verbose level")
8078
flag.BoolVar(&verbose, "verbose", false, "Enable verbose logging")
@@ -115,25 +113,32 @@ func main() {
115113
setupLog.Error(nil, "namespace cannot be empty")
116114
os.Exit(1)
117115
}
116+
118117
if cfg.Manager.SecretsNamespace == "" {
119118
setupLog.Error(nil, "secretsNamespace cannot be empty")
120119
os.Exit(1)
121120
}
122121

123122
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
124-
Scheme: scheme,
125-
MetricsBindAddress: cfg.Prometheus.Addr,
126-
Port: 9443,
127-
HealthProbeBindAddress: cfg.Health.Addr,
128-
LeaderElection: enableLeaderElection,
129-
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
130-
NewCache: cache.MultiNamespacedCacheBuilder([]string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}),
123+
Scheme: scheme,
124+
MetricsBindAddress: cfg.Prometheus.Addr,
125+
Port: 9443,
126+
HealthProbeBindAddress: cfg.Health.Addr,
127+
LeaderElection: true,
128+
LeaderElectionID: "ws-manager-mk2-leader.gitpod.io",
129+
LeaderElectionReleaseOnCancel: true,
130+
NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
131+
opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}
132+
return cache.New(config, opts)
133+
},
131134
})
132135
if err != nil {
133136
setupLog.Error(err, "unable to start manager")
134137
os.Exit(1)
135138
}
136139

140+
mgrCtx := ctrl.SetupSignalHandler()
141+
137142
maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
138143
if err != nil {
139144
setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
@@ -148,6 +153,21 @@ func main() {
148153
}
149154

150155
activity := activity.NewWorkspaceActivity()
156+
157+
go func() {
158+
for {
159+
select {
160+
case <-mgrCtx.Done():
161+
return
162+
case <-mgr.Elected():
163+
now := time.Now()
164+
setupLog.Info("updating activity started time", "now", now)
165+
activity.ManagerStartedAt = now
166+
return
167+
}
168+
}
169+
}()
170+
151171
timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
152172
if err != nil {
153173
setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
@@ -165,11 +185,13 @@ func main() {
165185
setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
166186
os.Exit(1)
167187
}
188+
168189
if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
169190
setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
170191
os.Exit(1)
171192
}
172-
if err = maintenanceReconciler.SetupWithManager(mgr); err != nil {
193+
194+
if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil {
173195
setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
174196
os.Exit(1)
175197
}
@@ -191,7 +213,7 @@ func main() {
191213
}
192214

193215
setupLog.Info("starting manager")
194-
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
216+
if err := mgr.Start(mgrCtx); err != nil {
195217
setupLog.Error(err, "problem running manager")
196218
os.Exit(1)
197219
}

‎components/ws-manager-mk2/service/manager.go

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,13 @@ import (
1616
validation "github.com/go-ozzo/ozzo-validation"
1717
"github.com/opentracing/opentracing-go"
1818
"github.com/prometheus/client_golang/prometheus"
19+
"github.com/sirupsen/logrus"
1920
"golang.org/x/xerrors"
2021
"google.golang.org/grpc/codes"
2122
"google.golang.org/grpc/peer"
2223
"google.golang.org/grpc/status"
2324
"google.golang.org/protobuf/proto"
2425
"google.golang.org/protobuf/types/known/timestamppb"
25-
26-
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
27-
"github.com/gitpod-io/gitpod/common-go/log"
28-
"github.com/gitpod-io/gitpod/common-go/tracing"
29-
"github.com/gitpod-io/gitpod/common-go/util"
30-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
31-
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
32-
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
33-
"github.com/gitpod-io/gitpod/ws-manager/api/config"
34-
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
35-
36-
csapi "github.com/gitpod-io/gitpod/content-service/api"
37-
"github.com/sirupsen/logrus"
3826
corev1 "k8s.io/api/core/v1"
3927
"k8s.io/apimachinery/pkg/api/errors"
4028
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +34,17 @@ import (
4634
"k8s.io/utils/pointer"
4735
"sigs.k8s.io/controller-runtime/pkg/client"
4836
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
37+
38+
wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
39+
"github.com/gitpod-io/gitpod/common-go/log"
40+
"github.com/gitpod-io/gitpod/common-go/tracing"
41+
"github.com/gitpod-io/gitpod/common-go/util"
42+
csapi "github.com/gitpod-io/gitpod/content-service/api"
43+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
44+
"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
45+
wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
46+
"github.com/gitpod-io/gitpod/ws-manager/api/config"
47+
workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
4948
)
5049

5150
const (

‎install/installer/pkg/components/ws-manager-mk2/deployment.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,18 @@
55
package wsmanagermk2
66

77
import (
8-
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
9-
"github.com/gitpod-io/gitpod/installer/pkg/common"
10-
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
11-
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
128
appsv1 "k8s.io/api/apps/v1"
139
corev1 "k8s.io/api/core/v1"
1410
"k8s.io/apimachinery/pkg/api/resource"
1511
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1612
"k8s.io/apimachinery/pkg/runtime"
1713
"k8s.io/apimachinery/pkg/util/intstr"
1814
"k8s.io/utils/pointer"
15+
16+
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
17+
"github.com/gitpod-io/gitpod/installer/pkg/common"
18+
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
19+
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
1920
)
2021

2122
func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
@@ -58,7 +59,6 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
5859
Name: Component,
5960
Args: []string{
6061
"--config", "/config/config.json",
61-
"--leader-elect",
6262
},
6363
Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerMk2.Version),
6464
ImagePullPolicy: corev1.PullIfNotPresent,
@@ -176,7 +176,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
176176
},
177177
Spec: appsv1.DeploymentSpec{
178178
Selector: &metav1.LabelSelector{MatchLabels: labels},
179-
Replicas: common.Replicas(ctx, Component),
179+
Replicas: pointer.Int32(2),
180180
Strategy: common.DeploymentStrategy,
181181
Template: corev1.PodTemplateSpec{
182182
ObjectMeta: metav1.ObjectMeta{

‎test/pkg/integration/setup.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ func Setup(ctx context.Context) (string, string, env.Environment, bool, string,
7979
assess string
8080
parallel bool
8181

82-
labels = make(flags.LabelsMap)
82+
labels = make(flags.LabelsMap)
83+
skipLabels = make(flags.LabelsMap)
8384
)
8485

8586
flagset := flag.CommandLine
@@ -99,6 +100,8 @@ func Setup(ctx context.Context) (string, string, env.Environment, bool, string,
99100
flagset.StringVar(&feature, "feature", "", "Regular expression that targets features to test")
100101
flagset.StringVar(&assess, "assess", "", "Regular expression that targets assertive steps to run")
101102
flagset.Var(&labels, "labels", "Comma-separated key/value pairs to filter tests by labels")
103+
flagset.Var(&skipLabels, "skip-labels", "Comma-separated key/value pairs to skip tests by labels")
104+
102105
if err := flagset.Parse(os.Args[1:]); err != nil {
103106
klog.Fatalf("cannot parse flags: %v", err)
104107
}
@@ -121,6 +124,7 @@ func Setup(ctx context.Context) (string, string, env.Environment, bool, string,
121124

122125
e.WithClient(client)
123126
e.WithLabels(labels)
127+
e.WithSkipLabels(skipLabels)
124128
e.WithNamespace(namespace)
125129

126130
// use the namespace from the CurrentContext

‎test/run.sh

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#
1515

1616
set -euo pipefail
17+
set -x
1718

1819
REPORT=""
1920
TEST_SUITE=all
@@ -114,7 +115,17 @@ if [ "$TEST_SUITE" == "workspace" ]; then
114115

115116
set +e
116117
# shellcheck disable=SC2086
117-
go test -p 10 -v $TEST_LIST "${args[@]}" -parallel-features=true 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy
118+
go test -p 4 -v $TEST_LIST "${args[@]}" -parallel-features=true -skip-labels="type=maintenance" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy
119+
RC=${PIPESTATUS[0]}
120+
set -e
121+
122+
if [ "${RC}" -ne "0" ]; then
123+
FAILURE_COUNT=$((FAILURE_COUNT+1))
124+
fi
125+
126+
set +e
127+
# shellcheck disable=SC2086
128+
go test -v $TEST_LIST "${args[@]}" -labels="type=maintenance" 2>&1
118129
RC=${PIPESTATUS[0]}
119130
set -e
120131

0 commit comments

Comments
 (0)