Skip to content

Make leader election lease duration configurable through manager options #412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"fmt"
"os"
"time"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -59,6 +60,50 @@ func Example() {
}
}

// This example creates a simple application Controller that is configured for ReplicaSets and Pods.
// This application controller will be running leader election with the provided configuration in the manager options.
// If leader election configuration is not provided, controller runs leader election with default values.
// Default values taken from: https://github.com/kubernetes/apiserver/blob/master/pkg/apis/config/v1alpha1/defaults.go
// defaultLeaseDuration = 15 * time.Second
// defaultRenewDeadline = 10 * time.Second
// defaultRetryPeriod = 2 * time.Second
//
// * Create a new application for ReplicaSets that manages Pods owned by the ReplicaSet and calls into
// ReplicaSetReconciler.
//
// * Start the application.
// TODO(pwittrock): Update this example when we have better dependency injection support
func Example_updateLeaderElectionDurations() {
var log = controllers.Log.WithName("builder-examples")
leaseDuration := 100 * time.Second
renewDeadline := 80 * time.Second
retryPeriod := 20 * time.Second
manager, err := controllers.NewManager(controllers.GetConfigOrDie(), controllers.Options{

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you instead create a new example to show how to change leader election lease duration?

LeaseDuration: &leaseDuration,
RenewDeadline: &renewDeadline,
RetryPeriod: &retryPeriod,
})
if err != nil {
log.Error(err, "could not create manager")
os.Exit(1)
}

err = controllers.
NewControllerManagedBy(manager). // Create the Controller
For(&appsv1.ReplicaSet{}). // ReplicaSet is the Application API
Owns(&corev1.Pod{}). // ReplicaSet owns Pods created by it
Complete(&ReplicaSetReconciler{Client: manager.GetClient()})
if err != nil {
log.Error(err, "could not create controller")
os.Exit(1)
}

if err := manager.Start(controllers.SetupSignalHandler()); err != nil {
log.Error(err, "could not start manager")
os.Exit(1)
}
}

// ReplicaSetReconciler is a simple Controller example implementation.
type ReplicaSetReconciler struct {
client.Client
Expand Down
27 changes: 21 additions & 6 deletions pkg/manager/internal.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ import (
"sigs.k8s.io/controller-runtime/pkg/webhook"
)

const (
// Values taken from: https://github.com/kubernetes/apiserver/blob/master/pkg/apis/config/v1alpha1/defaults.go
defaultLeaseDuration = 15 * time.Second
defaultRenewDeadline = 10 * time.Second
defaultRetryPeriod = 2 * time.Second
)

var log = logf.RuntimeLog.WithName("manager")

type controllerManager struct {
Expand Down Expand Up @@ -101,6 +108,16 @@ type controllerManager struct {
host string

webhookServer *webhook.Server

// leaseDuration is the duration that non-leader candidates will
// wait to force acquire leadership.
leaseDuration time.Duration
// renewDeadline is the duration that the acting master will retry
// refreshing leadership before giving up.
renewDeadline time.Duration
// retryPeriod is the duration the LeaderElector clients should wait
// between tries of actions.
retryPeriod time.Duration
}

// Add sets dependencies on i, and adds it to the list of runnables to start.
Expand Down Expand Up @@ -287,12 +304,10 @@ func (cm *controllerManager) start() {

func (cm *controllerManager) startLeaderElection() (err error) {
l, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
Lock: cm.resourceLock,
// Values taken from: https://github.com/kubernetes/apiserver/blob/master/pkg/apis/config/v1alpha1/defaults.go
// TODO(joelspeed): These timings should be configurable
LeaseDuration: 15 * time.Second,
RenewDeadline: 10 * time.Second,
RetryPeriod: 2 * time.Second,
Lock: cm.resourceLock,
LeaseDuration: cm.leaseDuration,
RenewDeadline: cm.renewDeadline,
RetryPeriod: cm.retryPeriod,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(_ context.Context) {
cm.start()
Expand Down
26 changes: 26 additions & 0 deletions pkg/manager/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,17 @@ type Options struct {
// will use for holding the leader lock.
LeaderElectionID string

// LeaseDuration is the duration that non-leader candidates will
// wait to force acquire leadership. This is measured against time of
// last observed ack. Default is 15 seconds.
LeaseDuration *time.Duration
// RenewDeadline is the duration that the acting master will retry
// refreshing leadership before giving up. Default is 10 seconds.
RenewDeadline *time.Duration
// RetryPeriod is the duration the LeaderElector clients should wait
// between tries of actions. Default is 2 seconds.
RetryPeriod *time.Duration

// Namespace if specified restricts the manager's cache to watch objects in
// the desired namespace Defaults to all namespaces
//
Expand Down Expand Up @@ -247,6 +258,9 @@ func New(config *rest.Config, options Options) (Manager, error) {
internalStopper: stop,
port: options.Port,
host: options.Host,
leaseDuration: *options.LeaseDuration,
renewDeadline: *options.RenewDeadline,
retryPeriod: *options.RetryPeriod,
}, nil
}

Expand Down Expand Up @@ -302,6 +316,18 @@ func setOptionsDefaults(options Options) Options {
if options.newMetricsListener == nil {
options.newMetricsListener = metrics.NewListener
}
leaseDuration, renewDeadline, retryPeriod := defaultLeaseDuration, defaultRenewDeadline, defaultRetryPeriod
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should pull the options.newResourceLock = leaderelection.NewResourceLock code fragment above closer to this block so that all leader election bits are in same place.

if options.LeaseDuration == nil {
options.LeaseDuration = &leaseDuration
}

if options.RenewDeadline == nil {
options.RenewDeadline = &renewDeadline
}

if options.RetryPeriod == nil {
options.RetryPeriod = &retryPeriod
}

return options
}