e2e tests for NLB instance mode

Author: kishorj

test/e2e/service/aws_resource_verifier.go

@@ -0,0 +1,184 @@
+package service
+
+import (
+	"context"
+	awssdk "github.com/aws/aws-sdk-go/aws"
+	elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"strconv"
+)
+
+type TargetGroupHC struct {
+	Protocol           string
+	Path               string
+	Port               string
+	Interval           int64
+	Timeout            int64
+	HealthyThreshold   int64
+	UnhealthyThreshold int64
+}
+
+type LoadBalancerExpectation struct {
+	Name          string
+	Type          string
+	Scheme        string
+	TargetType    string
+	Listeners     map[string]string // listener port, protocol
+	TargetGroups  map[string]string // target group port, protocol
+	NumTargets    int
+	TargetGroupHC *TargetGroupHC
+}
+
+func verifyAWSLoadBalancerResources(ctx context.Context, f *framework.Framework, lbARN string, expected LoadBalancerExpectation) error {
+	lb, err := f.LBManager.GetLoadBalancerFromARN(ctx, lbARN)
+	Expect(err).NotTo(HaveOccurred())
+	err = verifyLoadBalancerName(ctx, f, lb, expected.Name)
+	Expect(err).NotTo(HaveOccurred())
+	err = verifyLoadBalancerType(ctx, f, lb, expected.Type, expected.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+	err = verifyLoadBalancerListeners(ctx, f, lbARN, expected.Listeners)
+	Expect(err).NotTo(HaveOccurred())
+	err = verifyLoadBalancerTargetGroups(ctx, f, lbARN, expected)
+	Expect(err).NotTo(HaveOccurred())
+	return nil
+}
+
+func verifyLoadBalancerName(_ context.Context, f *framework.Framework, lb *elbv2sdk.LoadBalancer, lbName string) error {
+	if len(lbName) > 0 {
+		Expect(awssdk.StringValue(lb.LoadBalancerName)).To(Equal(lbName))
+	}
+	return nil
+}
+
+func verifyLoadBalancerType(_ context.Context, f *framework.Framework, lb *elbv2sdk.LoadBalancer, lbType, lbScheme string) error {
+	Expect(awssdk.StringValue(lb.Type)).To(Equal(lbType))
+	Expect(awssdk.StringValue(lb.Scheme)).To(Equal(lbScheme))
+	return nil
+}
+
+func verifyLoadBalancerAttributes(ctx context.Context, f *framework.Framework, lbARN string, expectedAttrs map[string]string) error {
+	lbAttrs, err := f.LBManager.GetLoadBalancerAttributes(ctx, lbARN)
+	Expect(err).NotTo(HaveOccurred())
+	for _, attr := range lbAttrs {
+		if val, ok := expectedAttrs[awssdk.StringValue(attr.Key)]; ok && val != awssdk.StringValue(attr.Value) {
+			return errors.Errorf("Attribute %v, expected %v, actual %v", awssdk.StringValue(attr.Key), val, awssdk.StringValue(attr.Value))
+		}
+	}
+	return nil
+}
+
+func verifyLoadBalancerTags(ctx context.Context, f *framework.Framework, lbARN string, expectedTags map[string]string) bool {
+	lbTags, err := f.LBManager.GetLoadBalancerTags(ctx, lbARN)
+	Expect(err).NotTo(HaveOccurred())
+	matchedTags := 0
+	for _, tag := range lbTags {
+		if val, ok := expectedTags[awssdk.StringValue(tag.Key)]; ok && val == awssdk.StringValue(tag.Value) {
+			matchedTags++
+		}
+	}
+	return matchedTags == len(expectedTags)
+}
+
+func getLoadBalancerListenerProtocol(ctx context.Context, f *framework.Framework, lbARN string, port string) string {
+	protocol := ""
+	listeners, err := f.LBManager.GetLoadBalancerListeners(ctx, lbARN)
+	Expect(err).ToNot(HaveOccurred())
+	for _, ls := range listeners {
+		if strconv.Itoa(int(awssdk.Int64Value(ls.Port))) == port {
+			protocol = awssdk.StringValue(ls.Protocol)
+		}
+	}
+	return protocol
+}
+
+func verifyLoadBalancerListeners(ctx context.Context, f *framework.Framework, lbARN string, listenersMap map[string]string) error {
+	listeners, err := f.LBManager.GetLoadBalancerListeners(ctx, lbARN)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(listeners)).To(Equal(len(listenersMap)))
+
+	for _, ls := range listeners {
+		portStr := strconv.Itoa(int(awssdk.Int64Value(ls.Port)))
+		Expect(listenersMap).Should(HaveKey(portStr))
+		Expect(awssdk.StringValue(ls.Protocol)).To(Equal(listenersMap[portStr]))
+	}
+	return nil
+}
+
+func verifyLoadBalancerTargetGroups(ctx context.Context, f *framework.Framework, lbARN string, expected LoadBalancerExpectation) error {
+	targetGroups, err := f.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(targetGroups)).To(Equal(len(expected.TargetGroups)))
+	for _, tg := range targetGroups {
+		Expect(awssdk.StringValue(tg.TargetType)).To(Equal(expected.TargetType))
+		Expect(awssdk.StringValue(tg.Protocol)).To(Equal(expected.TargetGroups[strconv.Itoa(int(awssdk.Int64Value(tg.Port)))]))
+		err = verifyTargetGroupHealthCheckConfig(tg, expected.TargetGroupHC)
+		Expect(err).NotTo(HaveOccurred())
+		err = verifyTargetGroupNumRegistered(ctx, f, awssdk.StringValue(tg.TargetGroupArn), expected.NumTargets)
+		Expect(err).NotTo(HaveOccurred())
+	}
+	return nil
+}
+
+func verifyTargetGroupHealthCheckConfig(tg *elbv2sdk.TargetGroup, hc *TargetGroupHC) error {
+	if hc != nil {
+		Expect(awssdk.StringValue(tg.HealthCheckProtocol)).To(Equal(hc.Protocol))
+		Expect(awssdk.StringValue(tg.HealthCheckPath)).To(Equal(hc.Path))
+		Expect(awssdk.StringValue(tg.HealthCheckPort)).To(Equal(hc.Port))
+		Expect(awssdk.Int64Value(tg.HealthCheckIntervalSeconds)).To(Equal(hc.Interval))
+		Expect(awssdk.Int64Value(tg.HealthCheckTimeoutSeconds)).To(Equal(hc.Timeout))
+		Expect(awssdk.Int64Value(tg.HealthyThresholdCount)).To(Equal(hc.HealthyThreshold))
+		Expect(awssdk.Int64Value(tg.UnhealthyThresholdCount)).To(Equal(hc.UnhealthyThreshold))
+	}
+	return nil
+}
+
+func verifyTargetGroupNumRegistered(ctx context.Context, f *framework.Framework, tgARN string, expectedTargets int) error {
+	if expectedTargets == 0 {
+		return nil
+	}
+	Eventually(func() bool {
+		numTargets, err := f.TGManager.GetCurrentTargetCount(ctx, tgARN)
+		Expect(err).ToNot(HaveOccurred())
+		return numTargets == expectedTargets
+	}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
+	return nil
+}
+
+func waitUntilTargetsAreHealthy(ctx context.Context, f *framework.Framework, lbARN string, expectedTargetCount int) error {
+	targetGroups, err := f.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(targetGroups)).To(Not(BeZero()))
+	// Check the first target group
+	tgARN := awssdk.StringValue(targetGroups[0].TargetGroupArn)
+
+	Eventually(func() (bool, error) {
+		return f.TGManager.CheckTargetGroupHealthy(ctx, tgARN, expectedTargetCount)
+	}, utils.PollTimeoutLong, utils.PollIntervalLong).Should(BeTrue())
+	return nil
+}
+
+func getTargetGroupHealthCheckProtocol(ctx context.Context, f *framework.Framework, lbARN string) string {
+	targetGroups, err := f.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
+	Expect(err).ToNot(HaveOccurred())
+	return awssdk.StringValue(targetGroups[0].HealthCheckProtocol)
+}
+
+func verifyTargetGroupAttributes(ctx context.Context, f *framework.Framework, lbARN string, expectedAttributes map[string]string) bool {
+	targetGroups, err := f.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(targetGroups)).To(Not(BeZero()))
+	// Check the first target group
+	tgARN := awssdk.StringValue(targetGroups[0].TargetGroupArn)
+	tgAttrs, err := f.TGManager.GetTargetGroupAttributes(ctx, tgARN)
+	Expect(err).NotTo(HaveOccurred())
+	matchedAttrs := 0
+	for _, attr := range tgAttrs {
+		if val, ok := expectedAttributes[awssdk.StringValue(attr.Key)]; ok && val == awssdk.StringValue(attr.Value) {
+			matchedAttrs++
+		}
+	}
+	return matchedAttrs == len(expectedAttributes)
+}

test/e2e/service/nlb_instance_target.go

@@ -0,0 +1,120 @@
+package service
+
+import (
+	"context"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
+)
+
+const (
+	defaultTestImage   = "kishorj/hello-multi:v1"
+	appContainerPort   = 80
+	defaultNumReplicas = 3
+	defaultName        = "instance-e2e"
+)
+
+type NLBInstanceTestStack struct {
+	resourceStack *resourceStack
+}
+
+func (s *NLBInstanceTestStack) Deploy(ctx context.Context, f *framework.Framework, svcAnnotations map[string]string) error {
+	dp := s.buildDeploymentSpec()
+	svc := s.buildServiceSpec(ctx, svcAnnotations)
+	s.resourceStack = NewResourceStack(dp, svc, "service-instance-e2e", false)
+
+	return s.resourceStack.Deploy(ctx, f)
+}
+
+func (s *NLBInstanceTestStack) UpdateServiceAnnotation(ctx context.Context, f *framework.Framework, svcAnnotations map[string]string) error {
+	return s.resourceStack.UpdateServiceAnnotations(ctx, f, svcAnnotations)
+}
+
+func (s *NLBInstanceTestStack) UpdateServiceTrafficPolicy(ctx context.Context, f *framework.Framework, trafficPolicy corev1.ServiceExternalTrafficPolicyType) error {
+	return s.resourceStack.UpdateServiceTrafficPolicy(ctx, f, trafficPolicy)
+}
+
+func (s *NLBInstanceTestStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error {
+	return s.resourceStack.ScaleDeployment(ctx, f, numReplicas)
+}
+
+func (s *NLBInstanceTestStack) Cleanup(ctx context.Context, f *framework.Framework) error {
+	return s.resourceStack.Cleanup(ctx, f)
+}
+
+func (s *NLBInstanceTestStack) GetLoadBalancerIngressHostName() string {
+	return s.resourceStack.GetLoadBalancerIngressHostname()
+}
+
+func (s *NLBInstanceTestStack) buildDeploymentSpec() *appsv1.Deployment {
+	numReplicas := int32(defaultNumReplicas)
+	labels := map[string]string{
+		"app.kubernetes.io/name":     "multi-port",
+		"app.kubernetes.io/instance": defaultName,
+	}
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: defaultName,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &numReplicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:            "app",
+							ImagePullPolicy: corev1.PullAlways,
+							Image:           defaultTestImage,
+							Ports: []corev1.ContainerPort{
+								{
+									ContainerPort: appContainerPort,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func (s *NLBInstanceTestStack) buildServiceSpec(ctx context.Context, annotations map[string]string) *corev1.Service {
+	labels := map[string]string{
+		"app.kubernetes.io/name":     "multi-port",
+		"app.kubernetes.io/instance": defaultName,
+	}
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: defaultName,
+			Annotations: map[string]string{
+				"service.beta.kubernetes.io/aws-load-balancer-type":        "external",
+				"service.beta.kubernetes.io/aws-load-balancer-target-type": "nlb-instance",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type:     corev1.ServiceTypeLoadBalancer,
+			Selector: labels,
+			Ports: []corev1.ServicePort{
+				{
+					Port:       80,
+					TargetPort: intstr.FromInt(80),
+					Protocol:   corev1.ProtocolTCP,
+				},
+			},
+		},
+	}
+	// Override annotations based on the argument
+	for key, value := range annotations {
+		svc.Annotations[key] = value
+	}
+
+	return svc
+}