Additional e2e tests for service

test/e2e/service/aws_resource_verifier.go

@@ -8,6 +8,7 @@ import (
 	"github.com/pkg/errors"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"sort"
 	"strconv"
 )
 
@@ -107,6 +108,34 @@ func verifyLoadBalancerListeners(ctx context.Context, f *framework.Framework, lb
 	return nil
 }
 
+func verifyLoadBalancerListenerCertificates(ctx context.Context, f *framework.Framework, lbARN string, expectedCertARNS []string) error {
+	listeners, err := f.LBManager.GetLoadBalancerListeners(ctx, lbARN)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(listeners)).Should(BeNumerically(">", 0))
+	listenerCerts, err := f.LBManager.GetLoadBalancerListenerCertificates(ctx, awssdk.StringValue(listeners[0].ListenerArn))
+	Expect(err).ToNot(HaveOccurred())
+
+	var observedCertArns []string
+	var defaultCert string
+	for _, cert := range listenerCerts {
+		if awssdk.BoolValue(cert.IsDefault) {
+			defaultCert = awssdk.StringValue(cert.CertificateArn)
+		}
+		observedCertArns = append(observedCertArns, awssdk.StringValue(cert.CertificateArn))
+	}
+	if defaultCert != expectedCertARNS[0] {
+		return errors.New("default cert does not match")
+	}
+	//Expect(defaultCert).To(Equal(expectedCertARNS[0]))
+	if len(expectedCertARNS) != len(observedCertArns) {
+		return errors.New("cert len mismatch")
+	}
+	sort.Strings(observedCertArns)
+	sort.Strings(expectedCertARNS)
+	Expect(expectedCertARNS).To(Equal(observedCertArns))
+	return nil
+}
+
 func verifyLoadBalancerTargetGroups(ctx context.Context, f *framework.Framework, lbARN string, expected LoadBalancerExpectation) error {
 	targetGroups, err := f.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
 	Expect(err).ToNot(HaveOccurred())

test/e2e/service/nlb_instance_target.go

@@ -6,7 +6,9 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -28,10 +30,14 @@ func (s *NLBInstanceTestStack) Deploy(ctx context.Context, f *framework.Framewor
 	return s.resourceStack.Deploy(ctx, f)
 }
 
-func (s *NLBInstanceTestStack) UpdateServiceAnnotation(ctx context.Context, f *framework.Framework, svcAnnotations map[string]string) error {
+func (s *NLBInstanceTestStack) UpdateServiceAnnotations(ctx context.Context, f *framework.Framework, svcAnnotations map[string]string) error {
 	return s.resourceStack.UpdateServiceAnnotations(ctx, f, svcAnnotations)
 }
 
+func (s *NLBInstanceTestStack) DeleteServiceAnnotations(ctx context.Context, f *framework.Framework, annotationKeys []string) error {
+	return s.resourceStack.DeleteServiceAnnotations(ctx, f, annotationKeys)
+}
+
 func (s *NLBInstanceTestStack) UpdateServiceTrafficPolicy(ctx context.Context, f *framework.Framework, trafficPolicy corev1.ServiceExternalTrafficPolicyType) error {
 	return s.resourceStack.UpdateServiceTrafficPolicy(ctx, f, trafficPolicy)
 }
@@ -48,6 +54,28 @@ func (s *NLBInstanceTestStack) GetLoadBalancerIngressHostName() string {
 	return s.resourceStack.GetLoadBalancerIngressHostname()
 }
 
+func (s *NLBInstanceTestStack) GetWorkerNodes(ctx context.Context, f *framework.Framework) ([]corev1.Node, error) {
+	nodeList := &corev1.NodeList{}
+	err := f.K8sClient.List(ctx, nodeList)
+	if err != nil {
+		return nil, err
+	}
+	return nodeList.Items, nil
+}
+
+func (s *NLBInstanceTestStack) ApplyNodeLabels(ctx context.Context, f *framework.Framework, node *corev1.Node, labels map[string]string) error {
+	f.Logger.Info("applying node labels", "node", k8s.NamespacedName(node))
+	oldNode := node.DeepCopy()
+	for key, value := range labels {
+		node.Labels[key] = value
+	}
+	if err := f.K8sClient.Patch(ctx, node, client.MergeFrom(oldNode)); err != nil {
+		f.Logger.Info("failed to update node", "node", k8s.NamespacedName(node))
+		return err
+	}
+	return nil
+}
+
 func (s *NLBInstanceTestStack) buildDeploymentSpec() *appsv1.Deployment {
 	numReplicas := int32(defaultNumReplicas)
 	labels := map[string]string{

test/e2e/service/nlb_instance_target_test.go

@@ -3,11 +3,13 @@ package service
 import (
 	"context"
 	"fmt"
+	awssdk "github.com/aws/aws-sdk-go/aws"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework/http"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"strings"
 )
 
 var _ = Describe("test k8s service reconciled by the aws load balancer controller", func() {
@@ -84,7 +86,7 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
 			})
 
 			By("enabling cross zone load balancing", func() {
-				err := stack.UpdateServiceAnnotation(ctx, tf, map[string]string{
+				err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
 					"service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled": "true",
 				})
 				Expect(err).NotTo(HaveOccurred())
@@ -97,7 +99,7 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
 			})
 
 			By("specifying load balancer tags", func() {
-				err := stack.UpdateServiceAnnotation(ctx, tf, map[string]string{
+				err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
 					"service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags": "instance-mode=true, key1=value1",
 				})
 				Expect(err).NotTo(HaveOccurred())
@@ -174,7 +176,7 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
 				Expect(err).NotTo(HaveOccurred())
 			})
 			By("specifying target group attributes annotation", func() {
-				err := stack.UpdateServiceAnnotation(ctx, tf, map[string]string{
+				err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
 					"service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": "preserve_client_ip.enabled=false, proxy_protocol_v2.enabled=true, deregistration_delay.timeout_seconds=120",
 				})
 				Expect(err).NotTo(HaveOccurred())
@@ -188,5 +190,144 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
 				}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
 			})
 		})
+		It("should create TLS listeners", func() {
+			if len(tf.Options.CertificateARNs) == 0 {
+				Skip("Skipping tests, certificates not specified")
+			}
+			By("deploying stack", func() {
+				err := stack.Deploy(ctx, tf, map[string]string{
+					"service.beta.kubernetes.io/aws-load-balancer-ssl-cert": tf.Options.CertificateARNs,
+				})
+				Expect(err).NotTo(HaveOccurred())
+			})
+			By("checking service status for lb dns name", func() {
+				dnsName = stack.GetLoadBalancerIngressHostName()
+				Expect(dnsName).ToNot(BeEmpty())
+			})
+
+			By("querying AWS loadbalancer from the dns name", func() {
+				var err error
+				lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(lbARN).ToNot(BeEmpty())
+			})
+			By("verifying AWS loadbalancer resources", func() {
+				err := verifyAWSLoadBalancerResources(ctx, tf, lbARN, LoadBalancerExpectation{
+					Type:       "network",
+					Scheme:     "internet-facing",
+					TargetType: "instance",
+					Listeners: map[string]string{
+						"80": "TLS",
+					},
+					TargetGroups: stack.resourceStack.getTargetGroupNodePortMap(),
+					NumTargets:   0,
+					TargetGroupHC: &TargetGroupHC{
+						Protocol:           "TCP",
+						Port:               "traffic-port",
+						Interval:           10,
+						Timeout:            10,
+						HealthyThreshold:   3,
+						UnhealthyThreshold: 3,
+					},
+				})
+				Expect(err).NotTo(HaveOccurred())
+			})
+			By("verifying listener certificates", func() {
+				expectedARNs := strings.Split(tf.Options.CertificateARNs, ",")
+				Eventually(func() bool {
+					return verifyLoadBalancerListenerCertificates(ctx, tf, lbARN, expectedARNs) == nil
+				}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
+			})
+			By("removing first certificate from annotation and updating the service", func() {
+				certs := strings.Split(tf.Options.CertificateARNs, ",")[1:]
+				if len(certs) == 0 {
+					return
+				}
+				err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
+					"service.beta.kubernetes.io/aws-load-balancer-ssl-cert": strings.Join(certs, ","),
+				})
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(func() bool {
+					return verifyLoadBalancerListenerCertificates(ctx, tf, lbARN, certs) == nil
+				}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
+			})
+		})
+		It("should enable proxy protocol v2", func() {
+			By("deploying stack", func() {
+				err := stack.Deploy(ctx, tf, map[string]string{
+					"service.beta.kubernetes.io/aws-load-balancer-proxy-protocol": "*",
+				})
+				Expect(err).ToNot(HaveOccurred())
+				dnsName = stack.GetLoadBalancerIngressHostName()
+				Expect(dnsName).ToNot(BeEmpty())
+				lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(lbARN).ToNot(BeEmpty())
+			})
+			By("verifying target group attributes", func() {
+				verified := verifyTargetGroupAttributes(ctx, tf, lbARN, map[string]string{
+					"proxy_protocol_v2.enabled": "true",
+				})
+				Expect(verified).To(BeTrue())
+			})
+			By("verifying precedence with target group attributes configuration", func() {
+				err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
+					"service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": "proxy_protocol_v2.enabled=false, deregistration_delay.timeout_seconds=120",
+				})
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(func() bool {
+					return verifyTargetGroupAttributes(ctx, tf, lbARN, map[string]string{
+						"proxy_protocol_v2.enabled":            "true",
+						"deregistration_delay.timeout_seconds": "120",
+					})
+				}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
+			})
+		})
+	})
+
+	Context("with NLB instance target configuration with target node labels", func() {
+		It("should add only the labelled nodes to the target group", func() {
+			By("deploying stack", func() {
+				err := stack.Deploy(ctx, tf, map[string]string{
+					"service.beta.kubernetes.io/aws-load-balancer-target-node-labels": "service.node.label/key1=value1",
+				})
+				Expect(err).ToNot(HaveOccurred())
+				dnsName = stack.GetLoadBalancerIngressHostName()
+				Expect(dnsName).ToNot(BeEmpty())
+				lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(lbARN).ToNot(BeEmpty())
+			})
+			By("applying label to 1 worker node", func() {
+				nodes, err := stack.GetWorkerNodes(ctx, tf)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(nodes)).To(BeNumerically(">", 0))
+				err = stack.ApplyNodeLabels(ctx, tf, &nodes[0], map[string]string{"service.node.label/key1": "value1"})
+				Expect(err).ToNot(HaveOccurred())
+
+				targetGroups, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(targetGroups)).To(Equal(1))
+				tgARN := awssdk.StringValue(targetGroups[0].TargetGroupArn)
+
+				err = verifyTargetGroupNumRegistered(ctx, tf, tgARN, 1)
+				Expect(err).ToNot(HaveOccurred())
+			})
+			By("removing target-node-labels annotation from the service", func() {
+				err := stack.DeleteServiceAnnotations(ctx, tf, []string{"service.beta.kubernetes.io/aws-load-balancer-target-node-labels"})
+				Expect(err).ToNot(HaveOccurred())
+
+				targetGroups, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(len(targetGroups)).To(Equal(1))
+				tgARN := awssdk.StringValue(targetGroups[0].TargetGroupArn)
+
+				nodes, err := stack.GetWorkerNodes(ctx, tf)
+				Expect(err).ToNot(HaveOccurred())
+
+				err = verifyTargetGroupNumRegistered(ctx, tf, tgARN, len(nodes))
+				Expect(err).ToNot(HaveOccurred())
+			})
+		})
 	})
 })

test/e2e/service/resource_stack.go

@@ -65,6 +65,16 @@ func (s *resourceStack) UpdateServiceAnnotations(ctx context.Context, f *framewo
 	return nil
 }
 
+func (s *resourceStack) DeleteServiceAnnotations(ctx context.Context, f *framework.Framework, annotationKeys []string) error {
+	if err := s.removeServiceAnnotations(ctx, f, annotationKeys); err != nil {
+		return err
+	}
+	if err := s.waitUntilServiceReady(ctx, f); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (s *resourceStack) UpdateServiceTrafficPolicy(ctx context.Context, f *framework.Framework, trafficPolicy corev1.ServiceExternalTrafficPolicyType) error {
 	if err := s.updateServiceTrafficPolicy(ctx, f, trafficPolicy); err != nil {
 		return err
@@ -142,6 +152,15 @@ func (s *resourceStack) updateServiceAnnotations(ctx context.Context, f *framewo
 	return s.updateService(ctx, f, oldSvc)
 }
 
+func (s *resourceStack) removeServiceAnnotations(ctx context.Context, f *framework.Framework, annotationKeys []string) error {
+	f.Logger.Info("removing service annotations", "svc", k8s.NamespacedName(s.svc))
+	oldSvc := s.svc.DeepCopy()
+	for _, key := range annotationKeys {
+		delete(s.svc.Annotations, key)
+	}
+	return s.updateService(ctx, f, oldSvc)
+}
+
 func (s *resourceStack) updateService(ctx context.Context, f *framework.Framework, oldSvc *corev1.Service) error {
 	f.Logger.Info("updating service", "svc", k8s.NamespacedName(s.svc))
 	if err := f.K8sClient.Patch(ctx, s.svc, client.MergeFrom(oldSvc)); err != nil {

test/framework/resources/aws/load_balancer.go

@@ -15,6 +15,7 @@ type LoadBalancerManager interface {
 	WaitUntilLoadBalancerAvailable(ctx context.Context, lbARN string) error
 	GetLoadBalancerFromARN(ctx context.Context, lbARN string) (*elbv2sdk.LoadBalancer, error)
 	GetLoadBalancerListeners(ctx context.Context, lbARN string) ([]*elbv2sdk.Listener, error)
+	GetLoadBalancerListenerCertificates(ctx context.Context, listnerARN string) ([]*elbv2sdk.Certificate, error)
 	GetLoadBalancerAttributes(ctx context.Context, lbARN string) ([]*elbv2sdk.LoadBalancerAttribute, error)
 	GetLoadBalancerTags(ctx context.Context, lbARN string) ([]*elbv2sdk.Tag, error)
 }
@@ -80,6 +81,12 @@ func (m *defaultLoadBalancerManager) GetLoadBalancerListeners(ctx context.Contex
 	return listeners.Listeners, nil
 }
 
+func (m *defaultLoadBalancerManager) GetLoadBalancerListenerCertificates(ctx context.Context, listnerARN string) ([]*elbv2sdk.Certificate, error) {
+	return m.elbv2Client.DescribeListenerCertificatesAsList(ctx, &elbv2sdk.DescribeListenerCertificatesInput{
+		ListenerArn: awssdk.String(listnerARN),
+	})
+}
+
 func (m *defaultLoadBalancerManager) GetLoadBalancerAttributes(ctx context.Context, lbARN string) ([]*elbv2sdk.LoadBalancerAttribute, error) {
 	resp, err := m.elbv2Client.DescribeLoadBalancerAttributesWithContext(ctx, &elbv2sdk.DescribeLoadBalancerAttributesInput{
 		LoadBalancerArn: awssdk.String(lbARN),