Add controller deployment ready logic to e2e script (#2658)

Add error handling

Author: cgchinmay

scripts/run-e2e-test.sh

@@ -51,6 +51,30 @@ kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/
 echo "Install aws-load-balancer-controller"
 helm upgrade -i aws-load-balancer-controller eks/aws-load-balancer-controller -n kube-system --set clusterName=$CLUSTER_NAME --set serviceAccount.create=false --set serviceAccount.name=aws-load-balancer-controller --set region=$REGION --set vpcId=$VPC_ID
 
+echo_time() {
+    date +"%D %T $*"
+}
+
+wait_until_deployment_ready() {
+	NS=""
+	if [ ! -z $2 ]; then
+		NS="-n $2"
+	fi
+	echo_time "Checking if deployment $1 is ready"
+	for i in $(seq 1 60); do
+		desiredReplicas=$(kubectl get deployments.apps $1 $NS -ojsonpath="{.spec.replicas}")
+		availableReplicas=$(kubectl get deployments.apps $1 $NS -ojsonpath="{.status.availableReplicas}")
+		if [[ ! -z $desiredReplicas && ! -z $availableReplicas && "$desiredReplicas" -eq "$availableReplicas" ]]; then
+			break
+		fi
+		echo -n "."
+		sleep 2
+	done 
+	echo_time "Deployment $1 $NS replicas desired=$desiredReplicas available=$availableReplicas"
+}
+
+wait_until_deployment_ready aws-load-balancer-controller kube-system
+
 function run_ginkgo_test() {
   local focus=$1
   echo "Starting the ginkgo tests from generated ginkgo test binaries with focus: $focus"

test/e2e/ingress/vanilla_ingress_test.go

@@ -102,7 +102,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				WithIngressClassName(ingClass.Name).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -138,7 +140,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -183,7 +187,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				WithIngressClassName(ingClass.Name).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			ExpectNoLBProvisionedForIngress(ctx, tf, ing)
@@ -212,7 +218,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			ExpectNoLBProvisionedForIngress(ctx, tf, ing)
@@ -240,7 +248,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			ExpectNoLBProvisionedForIngress(ctx, tf, ing)
@@ -275,7 +285,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -318,7 +330,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -389,7 +403,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-multiple-tg", PathType: &exact, Backend: ingForwardMultipleTGBackend}).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, dp1, svc1, dp2, svc2, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -503,7 +519,9 @@ var _ = Describe("vanilla ingress tests", func() {
 				AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path7", PathType: &exact, Backend: ingRulePath7Backend}).
 				WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
 			resStack := fixture.NewK8SResourceStack(tf, ing)
-			resStack.Setup(ctx)
+			err := resStack.Setup(ctx)
+			Expect(err).NotTo(HaveOccurred())
+
 			defer resStack.TearDown(ctx)
 
 			lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)