Skip to content

Commit f3c711c

Browse files
GrigoriyMikhalkinGrigoriyMikhalkin
committed
added leader election modes
1 parent 3dfb42d commit f3c711c

File tree

13 files changed

+80
-48
lines changed

13 files changed

+80
-48
lines changed

‎pkg/builder/controller_test.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"k8s.io/apimachinery/pkg/types"
3333
"sigs.k8s.io/controller-runtime/pkg/controller"
3434
"sigs.k8s.io/controller-runtime/pkg/handler"
35+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3536
"sigs.k8s.io/controller-runtime/pkg/manager"
3637
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3738
"sigs.k8s.io/controller-runtime/pkg/scheme"
@@ -63,7 +64,7 @@ var _ = Describe("application", func() {
6364
Owns(&appsv1.ReplicaSet{}).
6465
WithOptions(controller.Options{
6566
LeaderElection: &controller.LeaderElectionOptions{
66-
NeedLeaderElection: false,
67+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
6768
},
6869
}).
6970
Build(noop)
@@ -82,7 +83,7 @@ var _ = Describe("application", func() {
8283
Owns(&appsv1.ReplicaSet{}).
8384
WithOptions(controller.Options{
8485
LeaderElection: &controller.LeaderElectionOptions{
85-
NeedLeaderElection: false,
86+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
8687
},
8788
}).
8889
Build(noop)
@@ -109,7 +110,7 @@ var _ = Describe("application", func() {
109110
Owns(&appsv1.ReplicaSet{}).
110111
WithOptions(controller.Options{
111112
LeaderElection: &controller.LeaderElectionOptions{
112-
NeedLeaderElection: false,
113+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
113114
},
114115
}).
115116
Build(noop)
@@ -138,7 +139,7 @@ var _ = Describe("application", func() {
138139
WithOptions(controller.Options{
139140
MaxConcurrentReconciles: maxConcurrentReconciles,
140141
LeaderElection: &controller.LeaderElectionOptions{
141-
NeedLeaderElection: false,
142+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
142143
},
143144
}).
144145
Build(noop)
@@ -163,7 +164,7 @@ var _ = Describe("application", func() {
163164
Owns(&appsv1.ReplicaSet{}).
164165
WithOptions(controller.Options{
165166
LeaderElection: &controller.LeaderElectionOptions{
166-
NeedLeaderElection: false,
167+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
167168
},
168169
}).
169170
Build(noop)
@@ -176,7 +177,7 @@ var _ = Describe("application", func() {
176177
Owns(&appsv1.ReplicaSet{}).
177178
WithOptions(controller.Options{
178179
LeaderElection: &controller.LeaderElectionOptions{
179-
NeedLeaderElection: false,
180+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
180181
},
181182
}).
182183
Build(noop)
@@ -195,7 +196,7 @@ var _ = Describe("application", func() {
195196
Owns(&appsv1.ReplicaSet{}).
196197
WithOptions(controller.Options{
197198
LeaderElection: &controller.LeaderElectionOptions{
198-
NeedLeaderElection: false,
199+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
199200
},
200201
})
201202
doReconcileTest("3", stop, bldr, m, false)
@@ -210,7 +211,7 @@ var _ = Describe("application", func() {
210211
For(&appsv1.Deployment{}).
211212
WithOptions(controller.Options{
212213
LeaderElection: &controller.LeaderElectionOptions{
213-
NeedLeaderElection: false,
214+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
214215
},
215216
}).
216217
Watches( // Equivalent of Owns

‎pkg/cache/informer_cache.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"sigs.k8s.io/controller-runtime/pkg/cache/internal"
3131
"sigs.k8s.io/controller-runtime/pkg/client"
3232
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
33+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3334
)
3435

3536
var (
@@ -140,10 +141,10 @@ func (ip *informerCache) GetInformer(obj runtime.Object) (Informer, error) {
140141
return i.Informer, err
141142
}
142143

143-
// NeedLeaderElection implements the LeaderElectionRunnable interface
144+
// GetLeaderElectionMode implements the LeaderElectionRunnable interface
144145
// to indicate that this can be started without requiring the leader lock
145-
func (ip *informerCache) NeedLeaderElection() bool {
146-
return false
146+
func (ip *informerCache) GetLeaderElectionMode() leaderelection.Mode {
147+
return leaderelection.NonLeaderElectionMode
147148
}
148149

149150
// GetID implements the LeaderElectionRunnable interface.

‎pkg/cache/informer_cache_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"sigs.k8s.io/controller-runtime/pkg/cache"
1010
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
11+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
1112
"sigs.k8s.io/controller-runtime/pkg/manager"
1213
)
1314

@@ -23,6 +24,6 @@ var _ = Describe("informerCache", func() {
2324

2425
leaderElectionRunnable, ok := c.(manager.LeaderElectionRunnable)
2526
Expect(ok).To(BeTrue())
26-
Expect(leaderElectionRunnable.NeedLeaderElection()).To(BeFalse())
27+
Expect(leaderElectionRunnable.GetLeaderElectionMode()).To(Equal(leaderelection.NonLeaderElectionMode))
2728
})
2829
})

‎pkg/controller/controller.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"k8s.io/client-go/util/workqueue"
2323
"sigs.k8s.io/controller-runtime/pkg/handler"
2424
"sigs.k8s.io/controller-runtime/pkg/internal/controller"
25+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
2526
"sigs.k8s.io/controller-runtime/pkg/manager"
2627
"sigs.k8s.io/controller-runtime/pkg/predicate"
2728
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -42,8 +43,8 @@ type Options struct {
4243

4344
// Leader Election options
4445
type LeaderElectionOptions struct {
45-
// NeedLeaderElection determines whether or not to use leader election when starting the controller.
46-
NeedLeaderElection bool
46+
// LeaderElectionMode determines what leader election mode to use when starting the controller.
47+
LeaderElectionMode leaderelection.Mode
4748

4849
// LeaderElectionID determines the name of the configmap that leader election will use for holding the leader lock.
4950
LeaderElectionID string
@@ -86,8 +87,9 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error)
8687
}
8788

8889
if options.LeaderElection == nil {
90+
// Defaulting to per-manager leader election mode for backwards compatibility
8991
options.LeaderElection = &LeaderElectionOptions{
90-
NeedLeaderElection: true,
92+
LeaderElectionMode: leaderelection.PerManagerLeaderElectionMode,
9193
LeaderElectionID: name,
9294
}
9395
}
@@ -110,7 +112,7 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error)
110112
},
111113
MaxConcurrentReconciles: options.MaxConcurrentReconciles,
112114
Name: name,
113-
LeaderElection: options.LeaderElection.NeedLeaderElection,
115+
LeaderElectionMode: options.LeaderElection.LeaderElectionMode,
114116
LeaderElectionID: options.LeaderElection.LeaderElectionID,
115117
}
116118

‎pkg/controller/controller_integration_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"sigs.k8s.io/controller-runtime/pkg/cache"
2828
"sigs.k8s.io/controller-runtime/pkg/controller"
2929
"sigs.k8s.io/controller-runtime/pkg/handler"
30+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3031
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3132
"sigs.k8s.io/controller-runtime/pkg/source"
3233

@@ -65,7 +66,7 @@ var _ = Describe("controller", func() {
6566
return reconcile.Result{}, nil
6667
}),
6768
LeaderElection: &controller.LeaderElectionOptions{
68-
NeedLeaderElection: false,
69+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
6970
},
7071
})
7172
Expect(err).NotTo(HaveOccurred())

‎pkg/controller/controller_test.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
"sigs.k8s.io/controller-runtime/pkg/client"
2727
"sigs.k8s.io/controller-runtime/pkg/controller"
28+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
2829
"sigs.k8s.io/controller-runtime/pkg/manager"
2930
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3031
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
@@ -51,7 +52,7 @@ var _ = Describe("controller.Controller", func() {
5152
c, err := controller.New("", m, controller.Options{
5253
Reconciler: rec,
5354
LeaderElection: &controller.LeaderElectionOptions{
54-
NeedLeaderElection: false,
55+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
5556
},
5657
})
5758
Expect(c).To(BeNil())
@@ -78,7 +79,7 @@ var _ = Describe("controller.Controller", func() {
7879
c, err := controller.New("foo", m, controller.Options{
7980
Reconciler: &failRec{},
8081
LeaderElection: &controller.LeaderElectionOptions{
81-
NeedLeaderElection: false,
82+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
8283
},
8384
})
8485
Expect(c).To(BeNil())
@@ -95,7 +96,7 @@ var _ = Describe("controller.Controller", func() {
9596
c1, err := controller.New("c1", m, controller.Options{
9697
Reconciler: rec,
9798
LeaderElection: &controller.LeaderElectionOptions{
98-
NeedLeaderElection: false,
99+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
99100
},
100101
})
101102
Expect(err).NotTo(HaveOccurred())
@@ -104,7 +105,7 @@ var _ = Describe("controller.Controller", func() {
104105
c2, err := controller.New("c2", m, controller.Options{
105106
Reconciler: rec,
106107
LeaderElection: &controller.LeaderElectionOptions{
107-
NeedLeaderElection: false,
108+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
108109
},
109110
})
110111
Expect(err).NotTo(HaveOccurred())

‎pkg/internal/controller/controller.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"sigs.k8s.io/controller-runtime/pkg/handler"
3333
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
3434
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
35+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3536
"sigs.k8s.io/controller-runtime/pkg/predicate"
3637
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3738
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
@@ -50,9 +51,9 @@ type Controller struct {
5051
// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
5152
MaxConcurrentReconciles int
5253

53-
// LeaderElection determines whether or not to use leader election when
54+
// LeaderElectionMode determines which leader election mode to use when
5455
// starting the controller.
55-
LeaderElection bool
56+
LeaderElectionMode leaderelection.Mode
5657

5758
// LeaderElectionID determines the name of the configmap that leader election
5859
// will use for holding the leader lock.
@@ -305,8 +306,8 @@ func (c *Controller) updateMetrics(reconcileTime time.Duration) {
305306
ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds())
306307
}
307308

308-
func (c *Controller) NeedLeaderElection() bool {
309-
return c.LeaderElection
309+
func (c *Controller) GetLeaderElectionMode() leaderelection.Mode {
310+
return c.LeaderElectionMode
310311
}
311312

312313
func (c *Controller) GetID() string {

‎pkg/internal/recorder/recorder_integration_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
ref "k8s.io/client-go/tools/reference"
2626
"sigs.k8s.io/controller-runtime/pkg/controller"
2727
"sigs.k8s.io/controller-runtime/pkg/handler"
28+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
2829
"sigs.k8s.io/controller-runtime/pkg/manager"
2930
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3031
"sigs.k8s.io/controller-runtime/pkg/source"
@@ -62,7 +63,7 @@ var _ = Describe("recorder", func() {
6263
return reconcile.Result{}, nil
6364
}),
6465
LeaderElection: &controller.LeaderElectionOptions{
65-
NeedLeaderElection: false,
66+
LeaderElectionMode: leaderelection.NonLeaderElectionMode,
6667
},
6768
})
6869
Expect(err).NotTo(HaveOccurred())

‎pkg/leaderelection/leader_election.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,20 @@ import (
2929
"sigs.k8s.io/controller-runtime/pkg/recorder"
3030
)
3131

32-
const inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
32+
type Mode uint8
33+
34+
const (
35+
inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
36+
37+
// NonLeaderElectionMode mode for Runnables that don't need leader election
38+
NonLeaderElectionMode Mode = 0
39+
40+
// PerManagerLeaderElectionMode mode for Runnables that need per-manager leader election mode
41+
PerManagerLeaderElectionMode Mode = 1
42+
43+
// PerControllerGroupLeaderElectionMode mode for Runnables that need per-controller leader election mode
44+
PerControllerGroupLeaderElectionMode Mode = 2
45+
)
3346

3447
// Options provides the required configuration to create a new resource lock
3548
type Options struct {

‎pkg/manager/internal.go

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,9 @@ type controllerManager struct {
8989
// (and EventHandlers, Sources and Predicates).
9090
recorderProvider recorder.Provider
9191

92-
// defaultLeaderElection determines whether or not to use leader election by default
93-
// for runnables that don't implement LeaderElectionRunnable interface.
92+
// defaultLeaderElection determines whether or not to use leader election
93+
// for runnables that need per-manager leader election or
94+
// don't implement LeaderElectionRunnable interface.
9495
defaultLeaderElection bool
9596

9697
// defaultLeaderElectionID is used for runnables that don't implement LeaderElectionRunnable interface.
@@ -237,7 +238,16 @@ func (cm *controllerManager) Add(r Runnable) error {
237238
}
238239

239240
// Add the runnable to the leader election or the non-leaderelection list
240-
if leRunnable, ok := r.(LeaderElectionRunnable); (ok && !leRunnable.NeedLeaderElection()) || (!ok && !cm.defaultLeaderElection) {
241+
leRunnable, ok := r.(LeaderElectionRunnable)
242+
243+
// If runnable doesn't implement LeaderElectionRunnable interface and defaultLeaderElection is true
244+
// it's assumed that it needs per-manager leader election.
245+
// This is done to maintain backwards compatibility
246+
needPerManagerLE := cm.defaultLeaderElection && (!ok || (ok && (leRunnable.GetLeaderElectionMode() == crleaderelection.PerManagerLeaderElectionMode)))
247+
248+
needPerControllerLE := ok && (leRunnable.GetLeaderElectionMode() == crleaderelection.PerControllerGroupLeaderElectionMode)
249+
250+
if !needPerManagerLE && !needPerControllerLE {
241251
cm.nonLeaderElectionRunnables = append(cm.nonLeaderElectionRunnables, r)
242252

243253
if cm.started {
@@ -252,18 +262,15 @@ func (cm *controllerManager) Add(r Runnable) error {
252262
} else {
253263
var leID string
254264

255-
if ok {
265+
if needPerManagerLE {
266+
leID = cm.defaultLeaderElectionID
267+
} else {
256268
leID := leRunnable.GetID()
257269

258270
// Check that leader election ID is defined
259271
if leID == "" {
260272
return errors.New("LeaderElectionID must be configured")
261273
}
262-
} else if cm.defaultLeaderElection {
263-
// If runnable doesn't implement LeaderElectionRunnable interface and defaultLeaderElection is true
264-
// it's assumed that it needs leader election.
265-
// This is done to maintain backwards compatibility
266-
leID = cm.defaultLeaderElectionID
267274
}
268275

269276
cm.leaderElectionRunnables[leID] = append(cm.leaderElectionRunnables[leID], r)

‎pkg/manager/manager.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -119,12 +119,14 @@ type Options struct {
119119
// so that all controllers will not send list requests simultaneously.
120120
SyncPeriod *time.Duration
121121

122-
// DefaultLeaderElection determines whether or not to use leader election by default
123-
// for runnables that don't implement LeaderElectionRunnable interface.
122+
// DefaultLeaderElection determines whether or not to use leader election
123+
// for runnables that need per-manager leader election or
124+
// don't implement LeaderElectionRunnable interface.
124125
DefaultLeaderElection bool
125126

126127
// DefaultLeaderElectionID determines the name of the configmap that leader election
127-
// will use for runnables that don't implement LeaderElectionRunnable interface.
128+
// will use for runnables that need per-manager leader election or
129+
// don't implement LeaderElectionRunnable interface.
128130
// If not specified, default value will be assigned.
129131
DefaultLeaderElectionID string
130132

@@ -225,11 +227,10 @@ func (r RunnableFunc) Start(s <-chan struct{}) error {
225227

226228
// LeaderElectionRunnable knows if a Runnable needs to be run in the leader election mode.
227229
type LeaderElectionRunnable interface {
228-
// NeedLeaderElection returns true if the Runnable needs to be run in the leader election mode.
229-
// e.g. controllers need to be run in leader election mode, while webhook server doesn't.
230-
NeedLeaderElection() bool
230+
// GetLeaderElectionMode returns leader election mode in which Runnable needs to be run.
231+
GetLeaderElectionMode() leaderelection.Mode
231232

232-
// GetID returns leader election ID
233+
// GetID returns leader election ID for per-controller leader election mode.
233234
GetID() string
234235
}
235236

‎pkg/manager/manager_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"sigs.k8s.io/controller-runtime/pkg/cache"
3636
"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
3737
"sigs.k8s.io/controller-runtime/pkg/client"
38+
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
3839
fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake"
3940
"sigs.k8s.io/controller-runtime/pkg/metrics"
4041
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -903,8 +904,8 @@ func (*leRunnable) Start(<-chan struct{}) error {
903904
return nil
904905
}
905906

906-
func (*leRunnable) NeedLeaderElection() bool {
907-
return true
907+
func (*leRunnable) GetLeaderElectionMode() leaderelection.Mode {
908+
return leaderelection.PerControllerGroupLeaderElectionMode
908909
}
909910

910911
func (le *leRunnable) GetID() string {

0 commit comments

Comments
 (0)