Updated run-e2e-test script to remove eksctl iamserviceaccount creation (#2664)

* Updated run-e2e-test script to remove eksctl iamserviceaccount creation

* Updated run-e2e-test.sh to use service accounts created using aws cli

* Updated cleanup method

* Randomize Role Name

* Make TestId more unique

Author: cgchinmay

scripts/lib/common.sh

@@ -12,16 +12,13 @@ KUBE_CONFIG_PATH: $KUBE_CONFIG_PATH
 CLUSTER_NAME: $CLUSTER_NAME
 REGION: $REGION
 IP_FAMILY: $IP_FAMILY
-OS_OVERRIDE: $OS_OVERRIDE"
+OS_OVERRIDE: $OS_OVERRIDE
+ENDPOINT: $ENDPOINT"
 
 if [[ -z "${OS_OVERRIDE}" ]]; then
   OS_OVERRIDE=linux
 fi
 
-if [[ -n "${ENDPOINT}" ]]; then
-  ENDPOINT_FLAG="--endpoint $ENDPOINT"
-fi
-
 if [[ -z "${SKIP_MAKE_TEST_BINARIES}" ]]; then
   echo "making ginkgo test binaries"
   (cd $SCRIPT_DIR/../test && make build-test-binaries)

scripts/run-e2e-test.sh

@@ -18,29 +18,80 @@ function toggle_windows_scheduling(){
   done
 }
 
-echo "Cordon off windows nodes"
+TEST_ID=$(date +%s)
+echo "TEST_ID: $TEST_ID"
+ROLE_NAME="aws-load-balancer-controller-$TEST_ID"
+
+function cleanUp(){
+  # Need to recreae aws-load-balancer controller if we are updating SA
+  echo "delete aws-load-balancer-controller if exists"
+  helm delete aws-load-balancer-controller -n kube-system --timeout=10m || true
+ 
+  echo "delete service account if exists"
+  kubectl delete serviceaccount aws-load-balancer-controller -n kube-system --timeout 10m || true
+  
+  # IAM role and polcies are AWS Account specific, so need to clean them up if any from previous run
+  echo "detach IAM policy if it exists"
+  aws iam detach-role-policy --role-name $ROLE_NAME --policy-arn arn:aws:iam::$ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy || true
+
+  echo "delete $ROLE_NAME if it exists"
+  aws iam delete-role --role-name $ROLE_NAME || true
+
+  # Need to do this as last step
+  echo "delete AWSLoadBalancerControllerIAMPolicy if it exists"
+  aws iam delete-policy --policy-arn arn:aws:iam::$ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy || true
+}
+
+echo "cordon off windows nodes"
 toggle_windows_scheduling "cordon"
 
-eksctl utils associate-iam-oidc-provider \
-    --region $REGION \
-    --cluster $CLUSTER_NAME \
-    --approve
+echo "fetch OIDC provider"
+OIDC_PROVIDER=$(echo $CLUSTER_INFO |  jq -r '.cluster.identity.oidc.issuer' | sed -e "s/^https:\/\///")
+echo "OIDC Provider: $OIDC_PROVIDER"
+
+echo "create IAM policy document file"
+cat <<EOF > trust.json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::${ACCOUNT_ID}:oidc-provider/${OIDC_PROVIDER}"
+      },
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "${OIDC_PROVIDER}:aud": "sts.amazonaws.com",
+          "${OIDC_PROVIDER}:sub": "system:serviceaccount:kube-system:aws-load-balancer-controller"
+        }
+      }
+    }
+  ]
+}
+EOF
+
+echo "cleanup any stale resources from previous run"
+cleanUp
 
-echo "Creating AWSLoadbalancerController IAM Policy"
+echo "create Role with above policy document"
+aws iam create-role --role-name $ROLE_NAME --assume-role-policy-document file://trust.json --description "IAM Role to be used by aws-load-balancer-controller SA" || true
+
+echo "creating AWSLoadbalancerController IAM Policy"
 aws iam create-policy \
     --policy-name AWSLoadBalancerControllerIAMPolicy \
     --policy-document file://"$SCRIPT_DIR"/../docs/install/iam_policy.json || true
 
-echo "Creating IAM serviceaccount"
-eksctl create iamserviceaccount \
---cluster=$CLUSTER_NAME \
---namespace=kube-system \
---name=aws-load-balancer-controller \
---attach-policy-arn=arn:aws:iam::$ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy \
---override-existing-serviceaccounts \
---approve || true
+echo "attaching AWSLoadbalancerController IAM Policy to $ROLE_NAME"
+aws iam attach-role-policy --policy-arn arn:aws:iam::$ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy --role-name $ROLE_NAME || true
+
+echo "create service account"
+kubectl create serviceaccount aws-load-balancer-controller -n kube-system || true
+
+echo "annotate service account with $ROLE_NAME"
+kubectl annotate serviceaccount -n kube-system aws-load-balancer-controller eks.amazonaws.com/role-arn=arn:aws:iam::"$ACCOUNT_ID":role/"$ROLE_NAME" --overwrite=true || true
 
-echo "Update helm repo eks"
+echo "update helm repo eks"
 helm repo add eks https://aws.github.io/eks-charts
 
 helm repo update
@@ -97,20 +148,13 @@ run_ginkgo_test
 echo "Fetch most recent aws-load-balancer-controller logs"
 kubectl logs -l app.kubernetes.io/name=aws-load-balancer-controller --container aws-load-balancer-controller --tail=-1 -n kube-system
 
-echo "Delete aws-load-balancer-controller"
-helm delete aws-load-balancer-controller -n kube-system --timeout=10m || true
-
-echo "Delete iamserviceaccount"
-eksctl delete iamserviceaccount --name aws-load-balancer-controller --namespace kube-system --cluster $CLUSTER_NAME --timeout=10m || true
-
-echo "Delete TargetGroupBinding CRDs"
-kubectl delete -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master" --timeout=10m || true
-
 echo "Uncordon windows nodes"
 toggle_windows_scheduling "uncordon"
 
-# Need to do this as last step
-echo "Delete IAM Policy"
-aws iam delete-policy --policy-arn arn:aws:iam::$ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy || true
+echo "clean up resources from current run"
+cleanUp
+
+echo "Delete TargetGroupBinding CRDs if exists"
+kubectl delete -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master" --timeout=10m || true
 
 echo "Successfully finished the test suite $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds"