Skip to content

Commit 4942ef9

Browse files
cgchinmayTimothy-Dougherty
cgchinmay
authored and
Timothy-Dougherty
committed
Add controller deployment ready logic to e2e script (kubernetes-sigs#2658)
Add error handling
1 parent 5c3c36a commit 4942ef9

File tree

2 files changed

+143
-9
lines changed

2 files changed

+143
-9
lines changed

‎scripts/run-e2e-test.sh

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/bin/bash
2+
3+
# This script runs e2e tests on the AWS Load Balancer Controller
4+
5+
set -e
6+
7+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
8+
GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
9+
10+
source "$SCRIPT_DIR"/lib/common.sh
11+
12+
function toggle_windows_scheduling(){
13+
schedule=$1
14+
nodes=$(kubectl get nodes -l kubernetes.io/os=windows | tail -n +2 | cut -d' ' -f1)
15+
for n in $nodes
16+
do
17+
kubectl $schedule $n
18+
done
19+
}
20+
21+
echo "Cordon off windows nodes"
22+
toggle_windows_scheduling "cordon"
23+
24+
eksctl utils associate-iam-oidc-provider \
25+
--region $REGION \
26+
--cluster $CLUSTER_NAME \
27+
--approve
28+
29+
echo "Creating AWSLoadbalancerController IAM Policy"
30+
aws iam create-policy \
31+
--policy-name AWSLoadBalancerControllerIAMPolicy \
32+
--policy-document file://"$SCRIPT_DIR"/../docs/install/iam_policy.json || true
33+
34+
echo "Creating IAM serviceaccount"
35+
eksctl create iamserviceaccount \
36+
--cluster=$CLUSTER_NAME \
37+
--namespace=kube-system \
38+
--name=aws-load-balancer-controller \
39+
--attach-policy-arn=arn:aws:iam::$ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy \
40+
--override-existing-serviceaccounts \
41+
--approve || true
42+
43+
echo "Update helm repo eks"
44+
helm repo add eks https://aws.github.io/eks-charts
45+
46+
helm repo update
47+
48+
echo "Install TargetGroupBinding CRDs"
49+
kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master"
50+
51+
echo "Install aws-load-balancer-controller"
52+
helm upgrade -i aws-load-balancer-controller eks/aws-load-balancer-controller -n kube-system --set clusterName=$CLUSTER_NAME --set serviceAccount.create=false --set serviceAccount.name=aws-load-balancer-controller --set region=$REGION --set vpcId=$VPC_ID
53+
54+
echo_time() {
55+
date +"%D %T $*"
56+
}
57+
58+
wait_until_deployment_ready() {
59+
NS=""
60+
if [ ! -z $2 ]; then
61+
NS="-n $2"
62+
fi
63+
echo_time "Checking if deployment $1 is ready"
64+
for i in $(seq 1 60); do
65+
desiredReplicas=$(kubectl get deployments.apps $1 $NS -ojsonpath="{.spec.replicas}")
66+
availableReplicas=$(kubectl get deployments.apps $1 $NS -ojsonpath="{.status.availableReplicas}")
67+
if [[ ! -z $desiredReplicas && ! -z $availableReplicas && "$desiredReplicas" -eq "$availableReplicas" ]]; then
68+
break
69+
fi
70+
echo -n "."
71+
sleep 2
72+
done
73+
echo_time "Deployment $1 $NS replicas desired=$desiredReplicas available=$availableReplicas"
74+
}
75+
76+
wait_until_deployment_ready aws-load-balancer-controller kube-system
77+
78+
function run_ginkgo_test() {
79+
local focus=$1
80+
echo "Starting the ginkgo tests from generated ginkgo test binaries with focus: $focus"
81+
if [ "$IP_FAMILY" == "IPv4" ]; then
82+
(CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo $EXTRA_GINKGO_FLAGS --focus="$focus" -v --timeout 60m --fail-on-pending $GINKGO_TEST_BUILD/ingress.test -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID --ip-family=$IP_FAMILY || true)
83+
(CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo $EXTRA_GINKGO_FLAGS --focus="$focus" -v --timeout 60m --fail-on-pending $GINKGO_TEST_BUILD/service.test -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID --ip-family=$IP_FAMILY || true)
84+
elif [ "$IP_FAMILY" == "IPv6" ]; then
85+
(CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo $EXTRA_GINKGO_FLAGS --focus="$focus" --skip="instance" -v --timeout 60m --fail-on-pending $GINKGO_TEST_BUILD/ingress.test -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID --ip-family=$IP_FAMILY || true)
86+
(CGO_ENABLED=0 GOOS=$OS_OVERRIDE ginkgo $EXTRA_GINKGO_FLAGS --focus="$focus" --skip="instance" -v --timeout 60m --fail-on-pending $GINKGO_TEST_BUILD/service.test -- --kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID --ip-family=$IP_FAMILY || true)
87+
else
88+
echo "Invalid IP_FAMILY input, choose from IPv4 or IPv6 only"
89+
fi
90+
}
91+
92+
#Start the test
93+
run_ginkgo_test
94+
95+
# tail=-1 is added so that no logs are truncated
96+
# https://github.com/kubernetes/kubectl/issues/812
97+
echo "Fetch most recent aws-load-balancer-controller logs"
98+
kubectl logs -l app.kubernetes.io/name=aws-load-balancer-controller --container aws-load-balancer-controller --tail=-1 -n kube-system
99+
100+
echo "Delete aws-load-balancer-controller"
101+
helm delete aws-load-balancer-controller -n kube-system --timeout=10m || true
102+
103+
echo "Delete iamserviceaccount"
104+
eksctl delete iamserviceaccount --name aws-load-balancer-controller --namespace kube-system --cluster $CLUSTER_NAME --timeout=10m || true
105+
106+
echo "Delete TargetGroupBinding CRDs"
107+
kubectl delete -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master" --timeout=10m || true
108+
109+
echo "Uncordon windows nodes"
110+
toggle_windows_scheduling "uncordon"
111+
112+
# Need to do this as last step
113+
echo "Delete IAM Policy"
114+
aws iam delete-policy --policy-arn arn:aws:iam::$ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy || true
115+
116+
echo "Successfully finished the test suite $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds"

‎test/e2e/ingress/vanilla_ingress_test.go

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,9 @@ var _ = Describe("vanilla ingress tests", func() {
102102
WithIngressClassName(ingClass.Name).
103103
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
104104
resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing)
105-
resStack.Setup(ctx)
105+
err := resStack.Setup(ctx)
106+
Expect(err).NotTo(HaveOccurred())
107+
106108
defer resStack.TearDown(ctx)
107109

108110
lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -138,7 +140,9 @@ var _ = Describe("vanilla ingress tests", func() {
138140
AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
139141
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
140142
resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
141-
resStack.Setup(ctx)
143+
err := resStack.Setup(ctx)
144+
Expect(err).NotTo(HaveOccurred())
145+
142146
defer resStack.TearDown(ctx)
143147

144148
lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -183,7 +187,9 @@ var _ = Describe("vanilla ingress tests", func() {
183187
WithIngressClassName(ingClass.Name).
184188
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
185189
resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing)
186-
resStack.Setup(ctx)
190+
err := resStack.Setup(ctx)
191+
Expect(err).NotTo(HaveOccurred())
192+
187193
defer resStack.TearDown(ctx)
188194

189195
ExpectNoLBProvisionedForIngress(ctx, tf, ing)
@@ -212,7 +218,9 @@ var _ = Describe("vanilla ingress tests", func() {
212218
AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
213219
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
214220
resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
215-
resStack.Setup(ctx)
221+
err := resStack.Setup(ctx)
222+
Expect(err).NotTo(HaveOccurred())
223+
216224
defer resStack.TearDown(ctx)
217225

218226
ExpectNoLBProvisionedForIngress(ctx, tf, ing)
@@ -240,7 +248,9 @@ var _ = Describe("vanilla ingress tests", func() {
240248
AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
241249
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
242250
resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
243-
resStack.Setup(ctx)
251+
err := resStack.Setup(ctx)
252+
Expect(err).NotTo(HaveOccurred())
253+
244254
defer resStack.TearDown(ctx)
245255

246256
ExpectNoLBProvisionedForIngress(ctx, tf, ing)
@@ -275,7 +285,9 @@ var _ = Describe("vanilla ingress tests", func() {
275285
AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
276286
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
277287
resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
278-
resStack.Setup(ctx)
288+
err := resStack.Setup(ctx)
289+
Expect(err).NotTo(HaveOccurred())
290+
279291
defer resStack.TearDown(ctx)
280292

281293
lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -318,7 +330,9 @@ var _ = Describe("vanilla ingress tests", func() {
318330
AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}).
319331
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
320332
resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing)
321-
resStack.Setup(ctx)
333+
err := resStack.Setup(ctx)
334+
Expect(err).NotTo(HaveOccurred())
335+
322336
defer resStack.TearDown(ctx)
323337

324338
lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -389,7 +403,9 @@ var _ = Describe("vanilla ingress tests", func() {
389403
AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-multiple-tg", PathType: &exact, Backend: ingForwardMultipleTGBackend}).
390404
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
391405
resStack := fixture.NewK8SResourceStack(tf, dp1, svc1, dp2, svc2, ing)
392-
resStack.Setup(ctx)
406+
err := resStack.Setup(ctx)
407+
Expect(err).NotTo(HaveOccurred())
408+
393409
defer resStack.TearDown(ctx)
394410

395411
lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)
@@ -503,7 +519,9 @@ var _ = Describe("vanilla ingress tests", func() {
503519
AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path7", PathType: &exact, Backend: ingRulePath7Backend}).
504520
WithAnnotations(annotation).Build(sandboxNS.Name, "ing")
505521
resStack := fixture.NewK8SResourceStack(tf, ing)
506-
resStack.Setup(ctx)
522+
err := resStack.Setup(ctx)
523+
Expect(err).NotTo(HaveOccurred())
524+
507525
defer resStack.TearDown(ctx)
508526

509527
lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing)

0 commit comments

Comments
 (0)