enable e2e tests for service (kubernetes-sigs#1726)

Author: kishorj

scripts/ci_e2e_test.sh

@@ -214,12 +214,22 @@ test_controller_image() {
     return 1
   fi
 
-  ginkgo -v -r test/e2e/ingress/ -- \
+  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --region ${AWS_REGION} --query Account --output text)
+  S3_BUCKET=${S3_BUCKET:-"lb-controller-e2e-${AWS_ACCOUNT_ID}"}
+  CERTIFICATE_ARN_PREFIX=arn:aws:acm:${AWS_REGION}:${AWS_ACCOUNT_ID}:certificate
+  CERT_ID1="7caec311-1e1f-4b04-a061-bfa688fe813f"
+  CERT_ID2="724963dd-f571-4f2c-b549-5c7d0e35e4b8"
+  CERT_ID3="1001570b-1779-40c3-9b49-9a9a41e30058"
+  CERTIFICATE_ARNS=${CERTIFICATE_ARNS:-"${CERTIFICATE_ARN_PREFIX}/${CERT_ID1},${CERTIFICATE_ARN_PREFIX}/${CERT_ID2},${CERTIFICATE_ARN_PREFIX}/${CERT_ID3}"}
+
+  ginkgo -v -r test/e2e -- \
     --kubeconfig=${CLUSTER_KUBECONFIG} \
     --cluster-name=${CLUSTER_NAME} \
     --aws-region=${AWS_REGION} \
     --aws-vpc-id=${cluster_vpc_id} \
-    --controller-image=${CONTROLLER_IMAGE_NAME}
+    --controller-image=${CONTROLLER_IMAGE_NAME} \
+    --s3-bucket-name=${S3_BUCKET} \
+    --certificate-arns=${CERTIFICATE_ARNS}
 }
 
 #######################################

test/e2e/service/service.go

@@ -182,12 +182,14 @@ func (m *ServiceTest) VerifyLoadBalancerAttributes(ctx context.Context, f *frame
 }
 
 func (m *ServiceTest) CheckTargetGroupHealth(ctx context.Context, f *framework.Framework, tgArn string, numTargets int) (bool, error) {
-
 	resp, err := f.Cloud.ELBV2().DescribeTargetHealthWithContext(ctx, &elbv2.DescribeTargetHealthInput{
 		TargetGroupArn: aws.String(tgArn),
 	})
 	Expect(err).ToNot(HaveOccurred())
-	Expect(len(resp.TargetHealthDescriptions)).To(Equal(numTargets))
+
+	if len(resp.TargetHealthDescriptions) != numTargets {
+		return false, nil
+	}
 
 	healthy := true
 	for _, thd := range resp.TargetHealthDescriptions {
@@ -225,8 +227,6 @@ func (m *ServiceTest) CheckTargetGroups(ctx context.Context, f *framework.Framew
 		for _, tg := range targetGroups.TargetGroups {
 			Expect(aws.StringValue(tg.TargetType)).To(Equal(expected.TargetType))
 			Expect(aws.StringValue(tg.Protocol)).To(Equal(expected.TargetGroups[strconv.Itoa(int(aws.Int64Value(tg.Port)))]))
-			_, err := m.CheckTargetGroupHealth(ctx, f, aws.StringValue(tg.TargetGroupArn), expected.NumTargets)
-			Expect(err).ToNot(HaveOccurred())
 			err = m.verifyTargetGroupHealthCheckConfig(tg, expected.TargetGroupHC)
 			Expect(err).ToNot(HaveOccurred())
 		}

test/e2e/service/service_test.go

@@ -29,7 +29,7 @@ var _ = Describe("Service", func() {
 		labels      map[string]string
 	)
 
-	BeforeSuite(func() {
+	BeforeEach(func() {
 		ctx = context.Background()
 		name = utils.RandomDNS1123Label(20)
 		numReplicas = 3
@@ -75,7 +75,7 @@ var _ = Describe("Service", func() {
 		tf.DPManager.WaitUntilDeploymentReady(ctx, deployment)
 	})
 
-	AfterSuite(func() {
+	AfterEach(func() {
 		tf.K8sClient.Delete(ctx, deployment)
 		tf.DPManager.WaitUntilDeploymentDeleted(ctx, deployment)
 		tf.K8sClient.Delete(ctx, ns)
@@ -196,7 +196,6 @@ var _ = Describe("Service", func() {
 		var (
 			svcTest service.ServiceTest
 			svc     *corev1.Service
-			certArn string
 		)
 		BeforeEach(func() {
 			svc = &corev1.Service{
@@ -220,19 +219,13 @@ var _ = Describe("Service", func() {
 					},
 				},
 			}
-			certArn = svcTest.GenerateAndImportCertToACM(ctx, tf, "*.elb.us-west-2.amazonaws.com")
-			Expect(certArn).ToNot(BeNil())
 		})
-
-		AfterEach(func() {
-			Eventually(func() bool {
-				return svcTest.DeleteCertFromACM(ctx, tf, certArn) != nil
-			}, utils.PollTimeoutMedium, utils.PollIntervalLong).Should(BeTrue())
-		})
-
 		It("Should create TLS listeners", func() {
+			if len(tf.Options.CertificateARNs) == 0 {
+				Skip("Skipping tests, certificates not specified")
+			}
 			By("Creating service", func() {
-				svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-ssl-cert"] = certArn
+				svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-ssl-cert"] = tf.Options.CertificateARNs
 				err := svcTest.Create(ctx, tf, svc)
 				Expect(err).ToNot(HaveOccurred())
 			})
@@ -289,17 +282,20 @@ var _ = Describe("Service", func() {
 				}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
 			})
 			By("Specifying logging annotations", func() {
+				if len(tf.Options.S3BucketName) == 0 {
+					return
+				}
 				oldSvc := svc.DeepCopy()
 				svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-ssl-ports"] = "443, 333"
 				svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-access-log-enabled"] = "true"
-				svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name"] = "nlb-ip-svc-tls313"
+				svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name"] = tf.Options.S3BucketName
 				svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix"] = "nlb-pfx"
 				err := svcTest.Update(ctx, tf, svc, oldSvc)
 				Expect(err).ToNot(HaveOccurred())
 				Eventually(func() bool {
 					return svcTest.VerifyLoadBalancerAttributes(ctx, tf, map[string]string{
 						"access_logs.s3.enabled": "true",
-						"access_logs.s3.bucket":  "nlb-ip-svc-tls313",
+						"access_logs.s3.bucket":  tf.Options.S3BucketName,
 						"access_logs.s3.prefix":  "nlb-pfx",
 					}) == nil
 				}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())

test/framework/options.go

@@ -20,6 +20,10 @@ type Options struct {
 
 	// AWS Load Balancer Controller image. leave empty to use default one from helm chart.
 	ControllerImage string
+
+	// Additional parameters for e2e tests
+	S3BucketName    string
+	CertificateARNs string
 }
 
 func (options *Options) BindFlags() {
@@ -28,6 +32,9 @@ func (options *Options) BindFlags() {
 	flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `AWS VPC ID for the kubernetes cluster`)
 
 	flag.StringVar(&options.ControllerImage, "controller-image", "", `AWS Load Balancer Controller image`)
+
+	flag.StringVar(&options.S3BucketName, "s3-bucket-name", "", `S3 bucket to use for testing load balancer access logging feature`)
+	flag.StringVar(&options.CertificateARNs, "certificate-arns", "", `Certificate ARNs to use for TLS listeners`)
 }
 
 func (options *Options) Validate() error {