Add feature gates for ALB addons (kubernetes-sigs#1479)

* add feature gates and refactor command line flags

add aws-max-retries cmdline flag
add controller config options
add feature gates for waf, wafv2 and shield

* add sync-period flag

* specify config as pure data

add support for log-level flag
hoonor the ingress-class flag configuration
use AddonsConfig instead of FeatureGates

* refactor, get rid of inClusterConfig warning

* add webhook-bind-port and metrics-bind-addr flags

* cleanup unused imports

Author: kishorj

controllers/elbv2/targetgroupbinding_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/aws-load-balancer-controller/controllers/elbv2/eventhandlers"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/runtime"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding"
@@ -38,21 +39,23 @@ const (
 )
 
 // NewTargetGroupBindingReconciler constructs new targetGroupBindingReconciler
-func NewTargetGroupBindingReconciler(k8sClient client.Client, finalizerManager k8s.FinalizerManager, tgbResourceManager targetgroupbinding.ResourceManager, logger logr.Logger) *targetGroupBindingReconciler {
+func NewTargetGroupBindingReconciler(k8sClient client.Client, finalizerManager k8s.FinalizerManager, tgbResourceManager targetgroupbinding.ResourceManager, config config.ControllerConfig, logger logr.Logger) *targetGroupBindingReconciler {
 	return &targetGroupBindingReconciler{
-		k8sClient:          k8sClient,
-		finalizerManager:   finalizerManager,
-		tgbResourceManager: tgbResourceManager,
-		logger:             logger,
+		k8sClient:               k8sClient,
+		finalizerManager:        finalizerManager,
+		tgbResourceManager:      tgbResourceManager,
+		logger:                  logger,
+		maxConcurrentReconciles: config.TargetgroupBindingMaxConcurrentReconciles,
 	}
 }
 
 // targetGroupBindingReconciler reconciles a TargetGroupBinding object
 type targetGroupBindingReconciler struct {
-	k8sClient          client.Client
-	finalizerManager   k8s.FinalizerManager
-	tgbResourceManager targetgroupbinding.ResourceManager
-	logger             logr.Logger
+	k8sClient               client.Client
+	finalizerManager        k8s.FinalizerManager
+	tgbResourceManager      targetgroupbinding.ResourceManager
+	logger                  logr.Logger
+	maxConcurrentReconciles int
 }
 
 // +kubebuilder:rbac:groups=elbv2.k8s.aws,resources=targetgroupbindings,verbs=get;list;watch;update;patch;create;delete
@@ -117,7 +120,7 @@ func (r *targetGroupBindingReconciler) SetupWithManager(ctx context.Context, mgr
 		For(&elbv2api.TargetGroupBinding{}).
 		Watches(&source.Kind{Type: &corev1.Endpoints{}}, epEventsHandler).
 		Watches(&source.Kind{Type: &corev1.Node{}}, nodeEventsHandler).
-		WithOptions(controller.Options{MaxConcurrentReconciles: 3}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.maxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/ingress/group_controller.go

@@ -11,6 +11,7 @@ import (
 	"sigs.k8s.io/aws-load-balancer-controller/controllers/ingress/eventhandlers"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/ingress"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
@@ -30,7 +31,7 @@ const (
 
 // NewGroupReconciler constructs new GroupReconciler
 func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
-	networkingSGManager networkingpkg.SecurityGroupManager, networkingSGReconciler networkingpkg.SecurityGroupReconciler, clusterName string,
+	networkingSGManager networkingpkg.SecurityGroupManager, networkingSGReconciler networkingpkg.SecurityGroupReconciler, config config.ControllerConfig,
 	subnetsResolver networkingpkg.SubnetsResolver, logger logr.Logger) *groupReconciler {
 	annotationParser := annotations.NewSuffixAnnotationParser(ingressAnnotationPrefix)
 	authConfigBuilder := ingress.NewDefaultAuthConfigBuilder(annotationParser)
@@ -40,15 +41,17 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
 		cloud.EC2(), cloud.ACM(),
 		annotationParser, subnetsResolver,
 		authConfigBuilder, enhancedBackendBuilder,
-		cloud.VpcID(), clusterName, logger)
+		cloud.VpcID(), config.ClusterName, logger)
 	stackMarshaller := deploy.NewDefaultStackMarshaller()
-	stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, clusterName, ingressTagPrefix, logger)
-	groupLoader := ingress.NewDefaultGroupLoader(k8sClient, annotationParser, "alb")
+	stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, config.ClusterName, ingressTagPrefix, logger, config)
+	ingressConfig := config.IngressConfig
+	groupLoader := ingress.NewDefaultGroupLoader(k8sClient, annotationParser, ingressConfig.IngressClass)
 	k8sFinalizerManager := k8s.NewDefaultFinalizerManager(k8sClient, logger)
 	finalizerManager := ingress.NewDefaultFinalizerManager(k8sFinalizerManager)
 
 	return &groupReconciler{
 		k8sClient:        k8sClient,
+		ingressConfig:    ingressConfig,
 		eventRecorder:    eventRecorder,
 		groupLoader:      groupLoader,
 		finalizerManager: finalizerManager,
@@ -63,6 +66,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
 // GroupReconciler reconciles a ingress group
 type groupReconciler struct {
 	k8sClient        client.Client
+	ingressConfig    config.IngressConfig
 	eventRecorder    record.EventRecorder
 	referenceIndexer ingress.ReferenceIndexer
 	modelBuilder     ingress.ModelBuilder
@@ -152,7 +156,7 @@ func (r *groupReconciler) updateIngressStatus(ctx context.Context, ing *networki
 
 func (r *groupReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	c, err := controller.New(controllerName, mgr, controller.Options{
-		MaxConcurrentReconciles: 3,
+		MaxConcurrentReconciles: r.ingressConfig.MaxConcurrentReconciles,
 		Reconciler:              r,
 	})
 	if err != nil {

controllers/service/service_controller.go

@@ -9,6 +9,7 @@ import (
 	"sigs.k8s.io/aws-load-balancer-controller/controllers/service/eventhandlers"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
@@ -31,24 +32,26 @@ const (
 
 func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
 	sgManager networking.SecurityGroupManager, sgReconciler networking.SecurityGroupReconciler,
-	clusterName string, resolver networking.SubnetsResolver, logger logr.Logger) *serviceReconciler {
+	config config.ControllerConfig, resolver networking.SubnetsResolver, logger logr.Logger) *serviceReconciler {
 	annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix)
-	modelBuilder := nlb.NewDefaultModelBuilder(clusterName, resolver, annotationParser)
+	modelBuilder := nlb.NewDefaultModelBuilder(config.ClusterName, resolver, annotationParser)
 	return &serviceReconciler{
-		k8sClient:        k8sClient,
-		eventRecorder:    eventRecorder,
-		annotationParser: annotationParser,
-		finalizerManager: k8s.NewDefaultFinalizerManager(k8sClient, logger),
-		modelBuilder:     modelBuilder,
-		stackMarshaller:  deploy.NewDefaultStackMarshaller(),
-		stackDeployer:    deploy.NewDefaultStackDeployer(cloud, k8sClient, sgManager, sgReconciler, clusterName, serviceTagPrefix, logger),
-		logger:           logger,
+		k8sClient:               k8sClient,
+		eventRecorder:           eventRecorder,
+		annotationParser:        annotationParser,
+		finalizerManager:        k8s.NewDefaultFinalizerManager(k8sClient, logger),
+		modelBuilder:            modelBuilder,
+		stackMarshaller:         deploy.NewDefaultStackMarshaller(),
+		stackDeployer:           deploy.NewDefaultStackDeployer(cloud, k8sClient, sgManager, sgReconciler, config.ClusterName, serviceTagPrefix, logger, config),
+		logger:                  logger,
+		maxConcurrentReconciles: config.ServiceMaxConcurrentReconciles,
 	}
 }
 
 type serviceReconciler struct {
-	k8sClient     client.Client
-	eventRecorder record.EventRecorder
+	k8sClient               client.Client
+	eventRecorder           record.EventRecorder
+	maxConcurrentReconciles int
 
 	annotationParser annotations.Parser
 	finalizerManager k8s.FinalizerManager
@@ -145,7 +148,7 @@ func (r *serviceReconciler) updateServiceStatus(ctx context.Context, svc *corev1
 
 func (r *serviceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	c, err := controller.New(controllerName, mgr, controller.Options{
-		MaxConcurrentReconciles: 3,
+		MaxConcurrentReconciles: r.maxConcurrentReconciles,
 		Reconciler:              r,
 	})
 	if err != nil {

docs/guide/service/nlb_ip_mode.md

@@ -34,4 +34,4 @@ service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: "*"
 ```
 
 ## Security group
-NLB does not currently support a managed security group. For ingress access, the controller will resolve the security group for the ENI corresponding tho the endpoint pod. If the ENI has a single security group, it gets used. In case of multiple security gropus, the controller expects to find only one security group tagged with the Kubernetes cluster id. Controller will update the ingress rules on the security groups as per the service spec.
+NLB does not currently support a managed security group. For ingress access, the controller will resolve the security group for the ENI corresponding tho the endpoint pod. If the ENI has a single security group, it gets used. In case of multiple security groups, the controller expects to find only one security group tagged with the Kubernetes cluster id. Controller will update the ingress rules on the security groups as per the service spec.

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/prometheus/client_golang v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.5.1
+	go.uber.org/zap v1.10.0
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	gomodules.xyz/jsonpatch/v2 v2.0.1
 	k8s.io/api v0.18.4

main.go

@@ -18,8 +18,8 @@ package main
 
 import (
 	"context"
-	"flag"
 	"github.com/spf13/pflag"
+	zapraw "go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/aws-load-balancer-controller/controllers/service"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
 	inject "sigs.k8s.io/aws-load-balancer-controller/pkg/inject"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
@@ -42,14 +43,6 @@ import (
 	// +kubebuilder:scaffold:imports
 )
 
-const (
-	flagMetricsAddr          = "metrics-addr"
-	flagEnableLeaderElection = "enable-leader-election"
-	flagLeaderElectionID     = "leader-election-id"
-	flagK8sClusterName       = "cluster-name"
-	defaultLeaderElectionID  = "aws-load-balancer-controller-leader"
-)
-
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
@@ -63,49 +56,41 @@ func init() {
 }
 
 func main() {
-	var metricsAddr string
-	var enableLeaderElection bool
-	var k8sClusterName string
-	var leaderElectionID string
 	awsCloudConfig := aws.CloudConfig{ThrottleConfig: throttle.NewDefaultServiceOperationsThrottleConfig()}
 	injectConfig := inject.Config{}
+	controllerConfig := config.ControllerConfig{}
+
 	fs := pflag.NewFlagSet("", pflag.ExitOnError)
-	fs.StringVar(&metricsAddr, flagMetricsAddr, ":8080", "The address the metric endpoint binds to.")
-	fs.BoolVar(&enableLeaderElection, flagEnableLeaderElection, true,
-		"Enable leader election for controller manager. "+
-			"Enabling this will ensure there is only one active controller manager.")
-	fs.StringVar(&leaderElectionID, flagLeaderElectionID, defaultLeaderElectionID,
-		"Name of the leader election ID to use for this controller")
-	fs.StringVar(&k8sClusterName, flagK8sClusterName, "", "Kubernetes cluster name")
 	awsCloudConfig.BindFlags(fs)
 	injectConfig.BindFlags(fs)
-	fs.AddGoFlagSet(flag.CommandLine)
+	controllerConfig.BindFlags(fs)
+
 	if err := fs.Parse(os.Args); err != nil {
 		setupLog.Error(err, "invalid flags")
 		os.Exit(1)
 	}
-
-	ctrl.SetLogger(zap.New(zap.UseDevMode(false)))
-	if len(k8sClusterName) == 0 {
-		setupLog.Info("Kubernetes cluster name must be specified")
+	if err := controllerConfig.Validate(); err != nil {
+		setupLog.Error(err, "Failed to validate controller configuration")
 		os.Exit(1)
 	}
 
+	logLevel := zapraw.NewAtomicLevelAt(0)
+	if controllerConfig.LogLevel == "debug" {
+		logLevel = zapraw.NewAtomicLevelAt(-1)
+	}
+	ctrl.SetLogger(zap.New(zap.UseDevMode(false), zap.Level(&logLevel)))
+
 	cloud, err := aws.NewCloud(awsCloudConfig, metrics.Registry)
 	if err != nil {
 		setupLog.Error(err, "Unable to initialize AWS cloud")
 		os.Exit(1)
 	}
-
-	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
-
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		LeaderElection:     enableLeaderElection,
-		LeaderElectionID:   leaderElectionID,
-		Port:               9443,
-	})
+	restCfg, err := config.BuildRestConfig(controllerConfig.RuntimeConfig)
+	rtOpts := config.BuildRuntimeOptions(controllerConfig.RuntimeConfig, scheme)
+	if err != nil {
+		setupLog.Error(err, "Unable to build REST config")
+	}
+	mgr, err := ctrl.NewManager(restCfg, rtOpts)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
@@ -117,16 +102,16 @@ func main() {
 	sgReconciler := networking.NewDefaultSecurityGroupReconciler(sgManager, ctrl.Log)
 	finalizerManager := k8s.NewDefaultFinalizerManager(mgr.GetClient(), ctrl.Log)
 	tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(),
-		podENIResolver, nodeENIResolver, sgManager, sgReconciler, cloud.VpcID(), k8sClusterName, ctrl.Log)
+		podENIResolver, nodeENIResolver, sgManager, sgReconciler, cloud.VpcID(), controllerConfig.ClusterName, ctrl.Log)
 
-	subnetResolver := networking.NewSubnetsResolver(cloud.EC2(), cloud.VpcID(), k8sClusterName, ctrl.Log.WithName("subnets-resolver"))
+	subnetResolver := networking.NewSubnetsResolver(cloud.EC2(), cloud.VpcID(), controllerConfig.ClusterName, ctrl.Log.WithName("subnets-resolver"))
 	ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"),
-		sgManager, sgReconciler, k8sClusterName, subnetResolver,
+		sgManager, sgReconciler, controllerConfig, subnetResolver,
 		ctrl.Log.WithName("controllers").WithName("Ingress"))
 	svcReconciler := service.NewServiceReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("service"),
-		sgManager, sgReconciler, k8sClusterName, subnetResolver,
+		sgManager, sgReconciler, controllerConfig, subnetResolver,
 		ctrl.Log.WithName("controllers").WithName("Service"))
-	tgbReconciler := elbv2controller.NewTargetGroupBindingReconciler(mgr.GetClient(), finalizerManager, tgbResManager,
+	tgbReconciler := elbv2controller.NewTargetGroupBindingReconciler(mgr.GetClient(), finalizerManager, tgbResManager, controllerConfig,
 		ctrl.Log.WithName("controllers").WithName("TargetGroupBinding"))
 
 	ctx := context.Background()

pkg/aws/cloud.go

@@ -70,7 +70,7 @@ func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer) (Cloud,
 		cfg.VpcID = vpcId
 	}
 
-	awsCfg := aws.NewConfig().WithRegion(cfg.Region).WithSTSRegionalEndpoint(endpoints.RegionalSTSEndpoint)
+	awsCfg := aws.NewConfig().WithRegion(cfg.Region).WithSTSRegionalEndpoint(endpoints.RegionalSTSEndpoint).WithMaxRetries(cfg.MaxRetries)
 	sess = sess.Copy(awsCfg)
 	return &defaultCloud{
 		cfg:         cfg,

pkg/aws/cloud_config.go

@@ -6,24 +6,32 @@ import (
 )
 
 const (
-	flagAWSRegion      = "aws-region"
-	flagAWSAPIThrottle = "aws-api-throttle"
-	flagAWSVpcID       = "aws-vpc-id"
+	flagAWSRegion        = "aws-region"
+	flagAWSAPIThrottle   = "aws-api-throttle"
+	flagAWSVpcID         = "aws-vpc-id"
+	flagAWSMaxRetries    = "aws-max-retries"
+	defaultVpcID         = ""
+	defaultRegion        = ""
+	defaultAPIMaxRetries = 10
 )
 
 type CloudConfig struct {
 	// AWS Region for the kubernetes cluster
 	Region string
 
-	// Throttle settings for aws APIs
+	// Throttle settings for AWS APIs
 	ThrottleConfig *throttle.ServiceOperationsThrottleConfig
 
 	// VPC ID of the Kubernetes cluster
 	VpcID string
+
+	// Max retries configuration for AWS APIs
+	MaxRetries int
 }
 
 func (cfg *CloudConfig) BindFlags(fs *pflag.FlagSet) {
-	fs.StringVar(&cfg.Region, flagAWSRegion, "", "AWS Region for the kubernetes cluster")
+	fs.StringVar(&cfg.Region, flagAWSRegion, defaultRegion, "AWS Region for the kubernetes cluster")
 	fs.Var(cfg.ThrottleConfig, flagAWSAPIThrottle, "throttle settings for AWS APIs, format: serviceID1:operationRegex1=rate:burst,serviceID2:operationRegex2=rate:burst")
-	fs.StringVar(&cfg.VpcID, flagAWSVpcID, "", "AWS VPC ID for the Kubernetes cluster")
+	fs.StringVar(&cfg.VpcID, flagAWSVpcID, defaultVpcID, "AWS VPC ID for the Kubernetes cluster")
+	fs.IntVar(&cfg.MaxRetries, flagAWSMaxRetries, defaultAPIMaxRetries, "Maximum retries for AWS APIs")
 }

pkg/config/addons_config.go

@@ -0,0 +1,27 @@
+package config
+
+import "github.com/spf13/pflag"
+
+const (
+	flagWAFEnabled    = "enable-waf"
+	flagWAFV2Enabled  = "enable-wafv2"
+	flagShieldEnabled = "enable-shield"
+	defaultEnabled    = true
+)
+
+// AddonsConfig contains configuration for the addon features
+type AddonsConfig struct {
+	// WAF addon for ALB
+	WAFEnabled bool
+	// WAFV2 addon for ALB
+	WAFV2Enabled bool
+	// Shield addon for ALB
+	ShieldEnabled bool
+}
+
+// BindFlags binds the command line flags to the fields in the config object
+func (f *AddonsConfig) BindFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&f.WAFEnabled, flagWAFEnabled, defaultEnabled, "Enable WAF addon for ALB")
+	fs.BoolVar(&f.WAFEnabled, flagWAFV2Enabled, defaultEnabled, "Enable WAF V2 addon for ALB")
+	fs.BoolVar(&f.WAFEnabled, flagShieldEnabled, defaultEnabled, "Enable Shield addon for ALB")
+}