Added alb.ingress.kubernetes.io/routing-target annotation which when set to pod will configure the Target Group to use the pod ip and port for the targets.

Author: bigkraig

docs/ingress-resources.md

@@ -62,6 +62,7 @@ alb.ingress.kubernetes.io/healthcheck-timeout-seconds
 alb.ingress.kubernetes.io/healthy-threshold-count
 alb.ingress.kubernetes.io/unhealthy-threshold-count
 alb.ingress.kubernetes.io/listen-ports
+alb.ingress.kubernetes.io/routing-target
 alb.ingress.kubernetes.io/security-groups
 alb.ingress.kubernetes.io/subnets
 alb.ingress.kubernetes.io/success-codes
@@ -96,6 +97,8 @@ Optional annotations are:
 
 - **listen-ports**: Defines the ports the ALB will expose. It defaults to `[{"HTTP": 80}]` unless a certificate ARN is defined, then it is `[{"HTTPS": 443}]`. Uses a format as follows '[{"HTTP":8080,"HTTPS": 443}]'.
 
+- **routing-target**: Defines if the EC2 instance ID or the pod IP is added to the Target Groups. Defaults to `instance`. Valid options are `instance` and `pod`. With `instance` the Target Group targets are `<ec2 instance id>:<node port>`, for `pod` the targets are `<pod ip>:<pod port>`. When using the pod IP, it will route from all availabilty zones. `pod` is to be used when the pod network is routable and can be reached by the ALB.
+
 - **security-groups**: [Security groups](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_SecurityGroups.html) that should be applied to the ALB instance. These can be referenced by security group IDs or the name tag associated with each security group. Example ID values are `sg-723a380a,sg-a6181ede,sg-a5181edd`. Example tag values are `appSG, webSG`. When the annotation is not present, the controller will create a security group with appropriate ports allowing access to `0.0.0.0/0` and attached to the ALB. It will also create a security group for instances that allows all TCP traffic when the source is the security group created for the ALB.
 
 - **subnets**: The subnets where the ALB instance should be deployed. Must include 2 subnets, each in a different [availability zone](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html). These can be referenced by subnet IDs or the name tag associated with the subnet. Example values for subnet IDs are `subnet-a4f0098e,subnet-457ed533,subnet-95c904cd`. Example values for name tags are: `webSubnet,appSubnet`. If subnets are not specified the ALB controller will attempt to detect qualified subnets. This qualification is done by locating subnets that match the following criteria.
@@ -133,6 +136,7 @@ alb.ingress.kubernetes.io/healthcheck-protocol
 alb.ingress.kubernetes.io/healthcheck-timeout-seconds
 alb.ingress.kubernetes.io/healthy-threshold-count
 alb.ingress.kubernetes.io/unhealthy-threshold-count
+alb.ingress.kubernetes.io/routing-target
 alb.ingress.kubernetes.io/success-codes
 alb.ingress.kubernetes.io/target-group-attributes
 ```

pkg/alb/lb/loadbalancer.go

@@ -40,7 +40,7 @@ type NewDesiredLoadBalancerOptions struct {
 	IngressRules          []extensions.IngressRule
 	GetServiceNodePort    func(string, int32) (*int64, error)
 	GetServiceAnnotations func(string, string) (*map[string]string, error)
-	GetNodes              func() util.AWSStringSlice
+	TargetsFunc           func(*string, string, string, *int64) albelbv2.TargetDescriptions
 }
 
 // NewDesiredLoadBalancer returns a new loadbalancer.LoadBalancer based on the opts provided.
@@ -125,7 +125,7 @@ func NewDesiredLoadBalancer(o *NewDesiredLoadBalancerOptions) (newLoadBalancer *
 		GetServiceNodePort:    o.GetServiceNodePort,
 		GetServiceAnnotations: o.GetServiceAnnotations,
 		AnnotationFactory:     o.AnnotationFactory,
-		GetNodes:              o.GetNodes,
+		TargetsFunc:           o.TargetsFunc,
 	})
 
 	if err != nil {

pkg/alb/tg/targetgroup.go

@@ -7,16 +7,17 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albrgt"
+	api "k8s.io/api/core/v1"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/annotations"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albec2"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albelbv2"
+	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albrgt"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/util/log"
 	util "github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/util/types"
-	api "k8s.io/api/core/v1"
 )
 
 type NewDesiredTargetGroupOptions struct {
@@ -28,7 +29,7 @@ type NewDesiredTargetGroupOptions struct {
 	Logger         *log.Logger
 	Namespace      string
 	SvcName        string
-	Targets        util.AWSStringSlice
+	Targets        albelbv2.TargetDescriptions
 }
 
 // NewDesiredTargetGroup returns a new targetgroup.TargetGroup based on the parameters provided.
@@ -271,8 +272,8 @@ func (t *TargetGroup) modify(mods tgChange, rOpts *ReconcileOptions) error {
 
 	if mods&targetsModified != 0 {
 		t.logger.Infof("Modifying target group targets.")
-		additions := util.Difference(t.targets.desired, t.targets.current)
-		removals := util.Difference(t.targets.current, t.targets.desired)
+		additions := t.targets.desired.Difference(t.targets.current)
+		removals := t.targets.current.Difference(t.targets.desired)
 
 		// check/change targets
 		if len(additions) > 0 {
@@ -386,18 +387,10 @@ func (t *TargetGroup) needsModification() tgChange {
 }
 
 // Registers Targets (ec2 instances) to TargetGroup, must be called when Current != Desired
-func (t *TargetGroup) registerTargets(additions util.AWSStringSlice, rOpts *ReconcileOptions) error {
-	targets := []*elbv2.TargetDescription{}
-	for _, target := range additions {
-		targets = append(targets, &elbv2.TargetDescription{
-			Id:   target,
-			Port: t.tg.current.Port,
-		})
-	}
-
+func (t *TargetGroup) registerTargets(additions albelbv2.TargetDescriptions, rOpts *ReconcileOptions) error {
 	in := &elbv2.RegisterTargetsInput{
 		TargetGroupArn: t.CurrentARN(),
-		Targets:        targets,
+		Targets:        additions,
 	}
 
 	if _, err := albelbv2.ELBV2svc.RegisterTargets(in); err != nil {
@@ -420,18 +413,10 @@ func (t *TargetGroup) registerTargets(additions util.AWSStringSlice, rOpts *Reco
 }
 
 // Deregisters Targets (ec2 instances) from the TargetGroup, must be called when Current != Desired
-func (t *TargetGroup) deregisterTargets(removals util.AWSStringSlice, rOpts *ReconcileOptions) error {
-	targets := []*elbv2.TargetDescription{}
-	for _, target := range removals {
-		targets = append(targets, &elbv2.TargetDescription{
-			Id:   target,
-			Port: t.tg.current.Port,
-		})
-	}
-
+func (t *TargetGroup) deregisterTargets(removals albelbv2.TargetDescriptions, rOpts *ReconcileOptions) error {
 	in := &elbv2.DeregisterTargetsInput{
 		TargetGroupArn: t.CurrentARN(),
-		Targets:        targets,
+		Targets:        removals,
 	}
 
 	if _, err := albelbv2.ELBV2svc.DeregisterTargets(in); err != nil {
@@ -458,6 +443,6 @@ func (t *TargetGroup) CurrentARN() *string {
 	return t.tg.current.TargetGroupArn
 }
 
-func (t *TargetGroup) CurrentTargets() util.AWSStringSlice {
+func (t *TargetGroup) CurrentTargets() albelbv2.TargetDescriptions {
 	return t.targets.current
 }

pkg/alb/tg/targetgroups.go

@@ -129,7 +129,7 @@ type NewDesiredTargetGroupsOptions struct {
 	Logger                *log.Logger
 	GetServiceNodePort    func(string, int32) (*int64, error)
 	GetServiceAnnotations func(string, string) (*map[string]string, error)
-	GetNodes              func() util.AWSStringSlice
+	TargetsFunc           func(*string, string, string, *int64) albelbv2.TargetDescriptions
 }
 
 // NewDesiredTargetGroups returns a new targetgroups.TargetGroups based on an extensions.Ingress.
@@ -166,7 +166,7 @@ func NewDesiredTargetGroups(o *NewDesiredTargetGroupsOptions) (TargetGroups, err
 				Logger:         o.Logger,
 				Namespace:      o.Namespace,
 				SvcName:        path.Backend.ServiceName,
-				Targets:        o.GetNodes(),
+				Targets:        o.TargetsFunc(tgAnnotations.RoutingTarget, o.Namespace, path.Backend.ServiceName, port),
 			})
 
 			// If this target group is already defined, copy the current state to our new TG
@@ -178,14 +178,7 @@ func NewDesiredTargetGroups(o *NewDesiredTargetGroupsOptions) (TargetGroups, err
 
 				// If there is a current TG ARN we can use it to purge the desired targets of unready instances
 				if tg.CurrentARN() != nil {
-					targets := []*elbv2.TargetDescription{}
-					for _, instanceID := range targetGroup.targets.desired {
-						targets = append(targets, &elbv2.TargetDescription{
-							Id:   instanceID,
-							Port: port,
-						})
-					}
-					desired, err := albelbv2.ELBV2svc.DescribeTargetGroupTargetsForArn(tg.CurrentARN(), targets)
+					desired, err := albelbv2.ELBV2svc.DescribeTargetGroupTargetsForArn(tg.CurrentARN(), targetGroup.targets.desired)
 					if err != nil {
 						return nil, err
 					}

pkg/alb/tg/types.go

@@ -35,8 +35,8 @@ type attributes struct {
 }
 
 type targets struct {
-	current util.AWSStringSlice
-	desired util.AWSStringSlice
+	current albelbv2.TargetDescriptions
+	desired albelbv2.TargetDescriptions
 }
 
 type tags struct {

pkg/albingress/albingress.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albelbv2"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albrgt"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -56,7 +57,7 @@ type NewALBIngressFromIngressOptions struct {
 	ALBNamePrefix         string
 	GetServiceNodePort    func(string, int32) (*int64, error)
 	GetServiceAnnotations func(string, string) (*map[string]string, error)
-	GetNodes              func() util.AWSStringSlice
+	TargetsFunc           func(*string, string, string, *int64) albelbv2.TargetDescriptions
 	Recorder              record.EventRecorder
 	ConnectionIdleTimeout *int64
 	AnnotationFactory     annotations.AnnotationFactory
@@ -129,7 +130,7 @@ func NewALBIngressFromIngress(o *NewALBIngressFromIngressOptions) *ALBIngress {
 		CommonTags:            newIngress.Tags(o.ClusterName),
 		GetServiceNodePort:    o.GetServiceNodePort,
 		GetServiceAnnotations: o.GetServiceAnnotations,
-		GetNodes:              o.GetNodes,
+		TargetsFunc:           o.TargetsFunc,
 		AnnotationFactory:     o.AnnotationFactory,
 	})
 

pkg/albingress/albingress_test.go

@@ -3,9 +3,11 @@ package albingress
 import (
 	"testing"
 
+	"github.com/aws/aws-sdk-go/service/elbv2"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/annotations"
-	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/util/types"
+	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albelbv2"
 	"k8s.io/api/extensions/v1beta1"
 	extensions "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -71,10 +73,10 @@ func TestNewALBIngressFromIngress(t *testing.T) {
 			a := make(map[string]string)
 			return &a, nil
 		},
-		GetNodes: func() types.AWSStringSlice {
-			instance1 := "i-1"
-			instance2 := "i-2"
-			return types.AWSStringSlice{&instance1, &instance2}
+		TargetsFunc: func(*string, string, string, *int64) albelbv2.TargetDescriptions {
+			instance1 := &elbv2.TargetDescription{Id: aws.String("i-1")}
+			instance2 := &elbv2.TargetDescription{Id: aws.String("i-2")}
+			return albelbv2.TargetDescriptions{instance1, instance2}
 		},
 		ClusterName:   "testCluster",
 		ALBNamePrefix: "albNamePrefix",

pkg/albingress/albingresses.go

@@ -13,7 +13,6 @@ import (
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/annotations"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albelbv2"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albrgt"
-	util "github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/util/types"
 )
 
 // NewALBIngressesFromIngressesOptions are the options to NewALBIngressesFromIngresses
@@ -27,7 +26,7 @@ type NewALBIngressesFromIngressesOptions struct {
 	DefaultIngressClass   string
 	GetServiceNodePort    func(string, int32) (*int64, error)
 	GetServiceAnnotations func(string, string) (*map[string]string, error)
-	GetNodes              func() util.AWSStringSlice
+	TargetsFunc           func(*string, string, string, *int64) albelbv2.TargetDescriptions
 	AnnotationFactory     annotations.AnnotationFactory
 }
 
@@ -57,7 +56,7 @@ func NewALBIngressesFromIngresses(o *NewALBIngressesFromIngressesOptions) ALBIng
 			ALBNamePrefix:         o.ALBNamePrefix,
 			GetServiceNodePort:    o.GetServiceNodePort,
 			GetServiceAnnotations: o.GetServiceAnnotations,
-			GetNodes:              o.GetNodes,
+			TargetsFunc:           o.TargetsFunc,
 			Recorder:              o.Recorder,
 			AnnotationFactory:     o.AnnotationFactory,
 		})

pkg/annotations/annotations.go

@@ -47,6 +47,7 @@ const (
 	subnetsKey                    = "alb.ingress.kubernetes.io/subnets"
 	successCodesKey               = "alb.ingress.kubernetes.io/success-codes"
 	successCodesAltKey            = "alb.ingress.kubernetes.io/successCodes"
+	routingTargetKey              = "alb.ingress.kubernetes.io/routing-target"
 	tagsKey                       = "alb.ingress.kubernetes.io/tags"
 	ignoreHostHeader              = "alb.ingress.kubernetes.io/ignore-host-header"
 	targetGroupAttributesKey      = "alb.ingress.kubernetes.io/target-group-attributes"
@@ -72,6 +73,7 @@ type Annotations struct {
 	Ports                      []PortData
 	Scheme                     *string
 	IPAddressType              *string
+	RoutingTarget              *string
 	SecurityGroups             util.AWSStringSlice
 	Subnets                    util.Subnets
 	SuccessCodes               *string
@@ -149,6 +151,7 @@ func (vf *ValidatingAnnotationFactory) ParseAnnotations(opts *ParseAnnotationsOp
 		a.setPorts(annotations),
 		a.setScheme(annotations, opts.Namespace, opts.IngressName, vf.validator),
 		a.setIPAddressType(annotations),
+		a.setRoutingTarget(annotations),
 		a.setSecurityGroups(annotations, vf.validator),
 		a.setSubnets(annotations, *vf.clusterName, vf.validator),
 		a.setSuccessCodes(annotations),
@@ -393,6 +396,18 @@ func (a *Annotations) setIPAddressType(annotations map[string]string) error {
 	return nil
 }
 
+func (a *Annotations) setRoutingTarget(annotations map[string]string) error {
+	switch {
+	case annotations[routingTargetKey] == "":
+		a.RoutingTarget = aws.String("instance")
+		return nil
+	case annotations[routingTargetKey] != "instance" && annotations[routingTargetKey] != "pod":
+		return fmt.Errorf("ALB Routing Type [%v] must be either `instance` or `pod`", annotations[routingTargetKey])
+	}
+	a.RoutingTarget = aws.String(annotations[routingTargetKey])
+	return nil
+}
+
 func (a *Annotations) setSecurityGroups(annotations map[string]string, validator Validator) error {
 	// no security groups specified means controller should manage them, if so return and sg will be
 	// created and managed during reconcile.

pkg/aws/albec2/ec2.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/aws/albelbv2"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -203,10 +205,17 @@ func (e *EC2) DeleteSecurityGroupByID(sgID *string) error {
 
 // DisassociateSGFromInstanceIfNeeded loops through a list of instances to see if a managedSG
 // exists. If it does, it attempts to remove the managedSG from the list.
-func (e *EC2) DisassociateSGFromInstanceIfNeeded(instances []*string, managedSG *string) error {
+func (e *EC2) DisassociateSGFromInstanceIfNeeded(targetDescriptions albelbv2.TargetDescriptions, managedSG *string) error {
 	if managedSG == nil {
 		return fmt.Errorf("Managed SG passed was empty unable to disassociate from instances")
 	}
+
+	instances := targetDescriptions.InstanceIds()
+
+	if len(instances) < 1 {
+		return nil
+	}
+
 	in := &ec2.DescribeInstancesInput{
 		InstanceIds: instances,
 	}
@@ -264,7 +273,13 @@ func (e *EC2) DisassociateSGFromInstanceIfNeeded(instances []*string, managedSG
 
 // AssociateSGToInstanceIfNeeded loops through a list of instances to see if newSG exists
 // for them. It not, it is appended to the instances(s).
-func (e *EC2) AssociateSGToInstanceIfNeeded(instances []*string, newSG *string) error {
+func (e *EC2) AssociateSGToInstanceIfNeeded(targetDescriptions albelbv2.TargetDescriptions, newSG *string) error {
+	instances := targetDescriptions.InstanceIds()
+
+	if len(instances) < 1 {
+		return nil
+	}
+
 	in := &ec2.DescribeInstancesInput{
 		InstanceIds: instances,
 	}