Skip to content

Commit 0a1076a

Browse files
GrigoriyMikhalkinGrigoriyMikhalkin
committed
added leader election modes
1 parent ff25f2d commit 0a1076a

File tree

13 files changed

+80
-48
lines changed

13 files changed

+80
-48
lines changed

‎pkg/builder/controller_test.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"k8s.io/client-go/util/workqueue"
3434
"sigs.k8s.io/controller-runtime/pkg/controller"
3535
"sigs.k8s.io/controller-runtime/pkg/handler"
36+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3637
"sigs.k8s.io/controller-runtime/pkg/manager"
3738
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3839
"sigs.k8s.io/controller-runtime/pkg/scheme"
@@ -64,7 +65,7 @@ var _ = Describe("application", func() {
6465
Owns(&appsv1.ReplicaSet{}).
6566
WithOptions(controller.Options{
6667
LeaderElection: &controller.LeaderElectionOptions{
67-
NeedLeaderElection: false,
68+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
6869
},
6970
}).
7071
Build(noop)
@@ -83,7 +84,7 @@ var _ = Describe("application", func() {
8384
Owns(&appsv1.ReplicaSet{}).
8485
WithOptions(controller.Options{
8586
LeaderElection: &controller.LeaderElectionOptions{
86-
NeedLeaderElection: false,
87+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
8788
},
8889
}).
8990
Build(noop)
@@ -110,7 +111,7 @@ var _ = Describe("application", func() {
110111
Owns(&appsv1.ReplicaSet{}).
111112
WithOptions(controller.Options{
112113
LeaderElection: &controller.LeaderElectionOptions{
113-
NeedLeaderElection: false,
114+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
114115
},
115116
}).
116117
Build(noop)
@@ -139,7 +140,7 @@ var _ = Describe("application", func() {
139140
WithOptions(controller.Options{
140141
MaxConcurrentReconciles: maxConcurrentReconciles,
141142
LeaderElection: &controller.LeaderElectionOptions{
142-
NeedLeaderElection: false,
143+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
143144
},
144145
}).
145146
Build(noop)
@@ -186,7 +187,7 @@ var _ = Describe("application", func() {
186187
Owns(&appsv1.ReplicaSet{}).
187188
WithOptions(controller.Options{
188189
LeaderElection: &controller.LeaderElectionOptions{
189-
NeedLeaderElection: false,
190+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
190191
},
191192
}).
192193
Build(noop)
@@ -199,7 +200,7 @@ var _ = Describe("application", func() {
199200
Owns(&appsv1.ReplicaSet{}).
200201
WithOptions(controller.Options{
201202
LeaderElection: &controller.LeaderElectionOptions{
202-
NeedLeaderElection: false,
203+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
203204
},
204205
}).
205206
Build(noop)
@@ -218,7 +219,7 @@ var _ = Describe("application", func() {
218219
Owns(&appsv1.ReplicaSet{}).
219220
WithOptions(controller.Options{
220221
LeaderElection: &controller.LeaderElectionOptions{
221-
NeedLeaderElection: false,
222+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
222223
},
223224
})
224225
doReconcileTest("3", stop, bldr, m, false)
@@ -233,7 +234,7 @@ var _ = Describe("application", func() {
233234
For(&appsv1.Deployment{}).
234235
WithOptions(controller.Options{
235236
LeaderElection: &controller.LeaderElectionOptions{
236-
NeedLeaderElection: false,
237+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
237238
},
238239
}).
239240
Watches( // Equivalent of Owns

‎pkg/cache/informer_cache.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"sigs.k8s.io/controller-runtime/pkg/cache/internal"
3131
"sigs.k8s.io/controller-runtime/pkg/client"
3232
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
33+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3334
)
3435

3536
var (
@@ -163,10 +164,10 @@ func (ip *informerCache) GetInformer(obj runtime.Object) (Informer, error) {
163164
return i.Informer, err
164165
}
165166

166-
// NeedLeaderElection implements the LeaderElectionRunnable interface
167+
// GetLeaderElectionMode implements the LeaderElectionRunnable interface
167168
// to indicate that this can be started without requiring the leader lock
168-
func (ip *informerCache) NeedLeaderElection() bool {
169-
return false
169+
func (ip *informerCache) GetLeaderElectionMode() leaderelection.Mode {
170+
return leaderelection.NonLeaderElectionMode
170171
}
171172

172173
// GetID implements the LeaderElectionRunnable interface.

‎pkg/cache/informer_cache_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"sigs.k8s.io/controller-runtime/pkg/cache"
1010
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
11+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
1112
"sigs.k8s.io/controller-runtime/pkg/manager"
1213
)
1314

@@ -23,6 +24,6 @@ var _ = Describe("informerCache", func() {
2324

2425
leaderElectionRunnable, ok := c.(manager.LeaderElectionRunnable)
2526
Expect(ok).To(BeTrue())
26-
Expect(leaderElectionRunnable.NeedLeaderElection()).To(BeFalse())
27+
Expect(leaderElectionRunnable.GetLeaderElectionMode()).To(Equal(leaderelection.NonLeaderElectionMode))
2728
})
2829
})

‎pkg/controller/controller.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"k8s.io/client-go/util/workqueue"
2323
"sigs.k8s.io/controller-runtime/pkg/handler"
2424
"sigs.k8s.io/controller-runtime/pkg/internal/controller"
25+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
2526
"sigs.k8s.io/controller-runtime/pkg/manager"
2627
"sigs.k8s.io/controller-runtime/pkg/predicate"
2728
"sigs.k8s.io/controller-runtime/pkg/ratelimiter"
@@ -48,8 +49,8 @@ type Options struct {
4849

4950
// Leader Election options
5051
type LeaderElectionOptions struct {
51-
// NeedLeaderElection determines whether or not to use leader election when starting the controller.
52-
NeedLeaderElection bool
52+
// LeaderElectionMode determines what leader election mode to use when starting the controller.
53+
LeaderElectionMode leaderelection.Mode
5354

5455
// LeaderElectionID determines the name of the configmap that leader election will use for holding the leader lock.
5556
LeaderElectionID string
@@ -96,8 +97,9 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error)
9697
}
9798

9899
if options.LeaderElection == nil {
100+
// Defaulting to per-manager leader election mode for backwards compatibility
99101
options.LeaderElection = &LeaderElectionOptions{
100-
NeedLeaderElection: true,
102+
LeaderElectionMode: leaderelection.PerManagerLeaderElectionMode,
101103
LeaderElectionID: name,
102104
}
103105
}
@@ -120,7 +122,7 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error)
120122
},
121123
MaxConcurrentReconciles: options.MaxConcurrentReconciles,
122124
Name: name,
123-
LeaderElection: options.LeaderElection.NeedLeaderElection,
125+
LeaderElectionMode: options.LeaderElection.LeaderElectionMode,
124126
LeaderElectionID: options.LeaderElection.LeaderElectionID,
125127
}
126128

‎pkg/controller/controller_integration_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"sigs.k8s.io/controller-runtime/pkg/controller"
2929
"sigs.k8s.io/controller-runtime/pkg/controller/controllertest"
3030
"sigs.k8s.io/controller-runtime/pkg/handler"
31+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3132
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3233
"sigs.k8s.io/controller-runtime/pkg/source"
3334

@@ -66,7 +67,7 @@ var _ = Describe("controller", func() {
6667
return reconcile.Result{}, nil
6768
}),
6869
LeaderElection: &controller.LeaderElectionOptions{
69-
NeedLeaderElection: false,
70+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
7071
},
7172
})
7273
Expect(err).NotTo(HaveOccurred())

‎pkg/controller/controller_test.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
"sigs.k8s.io/controller-runtime/pkg/client"
2727
"sigs.k8s.io/controller-runtime/pkg/controller"
28+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
2829
"sigs.k8s.io/controller-runtime/pkg/manager"
2930
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3031
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
@@ -51,7 +52,7 @@ var _ = Describe("controller.Controller", func() {
5152
c, err := controller.New("", m, controller.Options{
5253
Reconciler: rec,
5354
LeaderElection: &controller.LeaderElectionOptions{
54-
NeedLeaderElection: false,
55+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
5556
},
5657
})
5758
Expect(c).To(BeNil())
@@ -78,7 +79,7 @@ var _ = Describe("controller.Controller", func() {
7879
c, err := controller.New("foo", m, controller.Options{
7980
Reconciler: &failRec{},
8081
LeaderElection: &controller.LeaderElectionOptions{
81-
NeedLeaderElection: false,
82+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
8283
},
8384
})
8485
Expect(c).To(BeNil())
@@ -95,7 +96,7 @@ var _ = Describe("controller.Controller", func() {
9596
c1, err := controller.New("c1", m, controller.Options{
9697
Reconciler: rec,
9798
LeaderElection: &controller.LeaderElectionOptions{
98-
NeedLeaderElection: false,
99+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
99100
},
100101
})
101102
Expect(err).NotTo(HaveOccurred())
@@ -104,7 +105,7 @@ var _ = Describe("controller.Controller", func() {
104105
c2, err := controller.New("c2", m, controller.Options{
105106
Reconciler: rec,
106107
LeaderElection: &controller.LeaderElectionOptions{
107-
NeedLeaderElection: false,
108+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
108109
},
109110
})
110111
Expect(err).NotTo(HaveOccurred())

‎pkg/internal/controller/controller.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"sigs.k8s.io/controller-runtime/pkg/handler"
3333
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
3434
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
35+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3536
"sigs.k8s.io/controller-runtime/pkg/predicate"
3637
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3738
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
@@ -50,9 +51,9 @@ type Controller struct {
5051
// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
5152
MaxConcurrentReconciles int
5253

53-
// LeaderElection determines whether or not to use leader election when
54+
// LeaderElectionMode determines which leader election mode to use when
5455
// starting the controller.
55-
LeaderElection bool
56+
LeaderElectionMode leaderelection.Mode
5657

5758
// LeaderElectionID determines the name of the configmap that leader election
5859
// will use for holding the leader lock.
@@ -305,8 +306,8 @@ func (c *Controller) updateMetrics(reconcileTime time.Duration) {
305306
ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds())
306307
}
307308

308-
func (c *Controller) NeedLeaderElection() bool {
309-
return c.LeaderElection
309+
func (c *Controller) GetLeaderElectionMode() leaderelection.Mode {
310+
return c.LeaderElectionMode
310311
}
311312

312313
func (c *Controller) GetID() string {

‎pkg/internal/recorder/recorder_integration_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
ref "k8s.io/client-go/tools/reference"
2626
"sigs.k8s.io/controller-runtime/pkg/controller"
2727
"sigs.k8s.io/controller-runtime/pkg/handler"
28+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
2829
"sigs.k8s.io/controller-runtime/pkg/manager"
2930
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3031
"sigs.k8s.io/controller-runtime/pkg/source"
@@ -62,7 +63,7 @@ var _ = Describe("recorder", func() {
6263
return reconcile.Result{}, nil
6364
}),
6465
LeaderElection: &controller.LeaderElectionOptions{
65-
NeedLeaderElection: false,
66+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
6667
},
6768
})
6869
Expect(err).NotTo(HaveOccurred())

‎pkg/leaderelection/leader_election.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,20 @@ import (
2929
"sigs.k8s.io/controller-runtime/pkg/recorder"
3030
)
3131

32-
const inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
32+
type Mode uint8
33+
34+
const (
35+
inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
36+
37+
// NonLeaderElectionMode mode for Runnables that don't need leader election
38+
NonLeaderElectionMode Mode = 0
39+
40+
// PerManagerLeaderElectionMode mode for Runnables that need per-manager leader election mode
41+
PerManagerLeaderElectionMode Mode = 1
42+
43+
// PerControllerGroupLeaderElectionMode mode for Runnables that need per-controller leader election mode
44+
PerControllerGroupLeaderElectionMode Mode = 2
45+
)
3346

3447
// Options provides the required configuration to create a new resource lock
3548
type Options struct {

‎pkg/manager/internal.go

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,9 @@ type controllerManager struct {
8989
// (and EventHandlers, Sources and Predicates).
9090
recorderProvider recorder.Provider
9191

92-
// defaultLeaderElection determines whether or not to use leader election by default
93-
// for runnables that don't implement LeaderElectionRunnable interface.
92+
// defaultLeaderElection determines whether or not to use leader election
93+
// for runnables that need per-manager leader election or
94+
// don't implement LeaderElectionRunnable interface.
9495
defaultLeaderElection bool
9596

9697
// defaultLeaderElectionID is used for runnables that don't implement LeaderElectionRunnable interface.
@@ -237,7 +238,16 @@ func (cm *controllerManager) Add(r Runnable) error {
237238
}
238239

239240
// Add the runnable to the leader election or the non-leaderelection list
240-
if leRunnable, ok := r.(LeaderElectionRunnable); (ok && !leRunnable.NeedLeaderElection()) || (!ok && !cm.defaultLeaderElection) {
241+
leRunnable, ok := r.(LeaderElectionRunnable)
242+
243+
// If runnable doesn't implement LeaderElectionRunnable interface and defaultLeaderElection is true
244+
// it's assumed that it needs per-manager leader election.
245+
// This is done to maintain backwards compatibility
246+
needPerManagerLE := cm.defaultLeaderElection && (!ok || (ok && (leRunnable.GetLeaderElectionMode() == crleaderelection.PerManagerLeaderElectionMode)))
247+
248+
needPerControllerLE := ok && (leRunnable.GetLeaderElectionMode() == crleaderelection.PerControllerGroupLeaderElectionMode)
249+
250+
if !needPerManagerLE && !needPerControllerLE {
241251
cm.nonLeaderElectionRunnables = append(cm.nonLeaderElectionRunnables, r)
242252

243253
if cm.started {
@@ -252,18 +262,15 @@ func (cm *controllerManager) Add(r Runnable) error {
252262
} else {
253263
var leID string
254264

255-
if ok {
265+
if needPerManagerLE {
266+
leID = cm.defaultLeaderElectionID
267+
} else {
256268
leID := leRunnable.GetID()
257269

258270
// Check that leader election ID is defined
259271
if leID == "" {
260272
return errors.New("LeaderElectionID must be configured")
261273
}
262-
} else if cm.defaultLeaderElection {
263-
// If runnable doesn't implement LeaderElectionRunnable interface and defaultLeaderElection is true
264-
// it's assumed that it needs leader election.
265-
// This is done to maintain backwards compatibility
266-
leID = cm.defaultLeaderElectionID
267274
}
268275

269276
cm.leaderElectionRunnables[leID] = append(cm.leaderElectionRunnables[leID], r)

‎pkg/manager/manager.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -119,12 +119,14 @@ type Options struct {
119119
// so that all controllers will not send list requests simultaneously.
120120
SyncPeriod *time.Duration
121121

122-
// DefaultLeaderElection determines whether or not to use leader election by default
123-
// for runnables that don't implement LeaderElectionRunnable interface.
122+
// DefaultLeaderElection determines whether or not to use leader election
123+
// for runnables that need per-manager leader election or
124+
// don't implement LeaderElectionRunnable interface.
124125
DefaultLeaderElection bool
125126

126127
// DefaultLeaderElectionID determines the name of the configmap that leader election
127-
// will use for runnables that don't implement LeaderElectionRunnable interface.
128+
// will use for runnables that need per-manager leader election or
129+
// don't implement LeaderElectionRunnable interface.
128130
// If not specified, default value will be assigned.
129131
DefaultLeaderElectionID string
130132

@@ -225,11 +227,10 @@ func (r RunnableFunc) Start(s <-chan struct{}) error {
225227

226228
// LeaderElectionRunnable knows if a Runnable needs to be run in the leader election mode.
227229
type LeaderElectionRunnable interface {
228-
// NeedLeaderElection returns true if the Runnable needs to be run in the leader election mode.
229-
// e.g. controllers need to be run in leader election mode, while webhook server doesn't.
230-
NeedLeaderElection() bool
230+
// GetLeaderElectionMode returns leader election mode in which Runnable needs to be run.
231+
GetLeaderElectionMode() leaderelection.Mode
231232

232-
// GetID returns leader election ID
233+
// GetID returns leader election ID for per-controller leader election mode.
233234
GetID() string
234235
}
235236

‎pkg/manager/manager_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"sigs.k8s.io/controller-runtime/pkg/cache"
3636
"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
3737
"sigs.k8s.io/controller-runtime/pkg/client"
38+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3839
fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake"
3940
"sigs.k8s.io/controller-runtime/pkg/metrics"
4041
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -903,8 +904,8 @@ func (*leRunnable) Start(<-chan struct{}) error {
903904
return nil
904905
}
905906

906-
func (*leRunnable) NeedLeaderElection() bool {
907-
return true
907+
func (*leRunnable) GetLeaderElectionMode() leaderelection.Mode {
908+
return leaderelection.PerControllerGroupLeaderElectionMode
908909
}
909910

910911
func (le *leRunnable) GetID() string {

0 commit comments

Comments
 (0)