Merge pull request #44 from aws-observability/edit-retry-mechanism

Edit retry mechanism

Author: harrryr

.github/workflows/actions/execute_and_retry/action.yml

@@ -6,11 +6,11 @@ inputs:
   pre-command:
     required: false
     type: string
-  # (Optional) Number of retries to perform. Default is 2
+  # (Optional) Number of retries to perform. Default is 3
   max_retry:
     required: false
     type: number
-    default: 2
+    default: 3
   # (Required) Command to execute with the retry mechanism
   command:
     required: true
@@ -19,6 +19,11 @@ inputs:
   cleanup:
     required: false
     type: string
+  # (Optional) Time to wait between each attempt in seconds. Default is 10 seconds
+  sleep_time:
+    required: false
+    type: number
+    default: 10
   # (Optional) Follow-up command after the main command is finished.
   post-command:
     required: false
@@ -35,26 +40,35 @@ runs:
         COMMAND: ${{ inputs.command }}
         CLEANUP: ${{ inputs.cleanup }}
         POST_COMMAND: ${{ inputs.post-command }}
+        SLEEP_TIME: ${{ inputs.sleep_time }}
       run: |
+        echo "Starting the execute_and_retry action for command $COMMAND"
+        echo "Executing pre-command for the execute_and_retry action"
         eval "$PRE_COMMAND"
         
         retry_counter=0
         while [ $retry_counter -lt $MAX_RETRY ]; do
+           echo "Attempt Number $retry_counter for execute_and_retry action"
+
           attempt_failed=0
           eval "$COMMAND" || attempt_failed=$?
   
           if [ $attempt_failed -ne 0 ]; then
+            echo "Command failed for execute_and_retry action, executing cleanup command for another attempt"
+
             eval "$CLEANUP"
             retry_counter=$(($retry_counter+1))
-            sleep 5
+            sleep "$SLEEP_TIME"
           else
+            echo "Command executed successfully for execute_and_retry"
             break
           fi
-  
-          if [ $retry_counter -eq $max_retry ]; then
-            echo "Max retry reached, command failed to execute properly. Exiting code"
+          if [[ $retry_counter -ge $MAX_RETRY ]]; then
+            echo "Max retry reached, command failed to execute properly. Exiting action"
             exit 1
           fi
         done
 
-        eval "$POST_COMMAND"
+        echo "Executing post-command for the execute_and_retry action"
+        eval "$POST_COMMAND"
+        echo "Exiting execute_and_retry action"

.github/workflows/appsignals-e2e-eks-test.yml

@@ -108,6 +108,12 @@ jobs:
           --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \
           --region ${{ inputs.aws-region }} \
           --approve"
+          cleanup: "eksctl delete iamserviceaccount \
+          --name service-account-${{ env.TESTING_ID }} \
+          --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \
+          --cluster ${{ inputs.test-cluster-name }} \
+          --region ${{ inputs.aws-region }}"
+          sleep_time: 60
 
       - name: Initiate Terraform
         uses: ./.github/workflows/actions/execute_and_retry
@@ -148,7 +154,7 @@ jobs:
             # after installing App Signals. Attempts to connect will be made for up to 10 minutes
             if [ $deployment_failed -eq 0 ]; then
               . ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-              execute_and_retry 2 \
+              execute_and_retry 3 \
               "${{ env.TEST_RESOURCES_FOLDER }}/enablement-script/enable-app-signals.sh \
               ${{ inputs.test-cluster-name }} \
               ${{ inputs.aws-region }} \
@@ -157,10 +163,11 @@ jobs:
               ${{ inputs.test-cluster-name }} \
               ${{ inputs.aws-region }} \
               ${{ env.SAMPLE_APP_NAMESPACE }} && \
-              aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}"
+              aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}" \
+              60
           
-              execute_and_retry 2 "kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }}"
-              execute_and_retry 2 "kubectl wait --for=condition=Ready --request-timeout '5m' pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }}"
+              execute_and_retry 2 "kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }}" "" 60
+              execute_and_retry 2 "kubectl wait --for=condition=Ready --request-timeout '5m' pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }}" "" 10
           
               echo "Attempting to connect to the main sample app endpoint"
               main_sample_app_endpoint=http://$(terraform output sample_app_endpoint)
@@ -224,7 +231,7 @@ jobs:
               break
             fi
           
-            if [ $retry_counter -eq $max_retry ]; then
+            if [ $retry_counter -ge $max_retry ]; then
               echo "Max retry reached, failed to deploy terraform and connect to the endpoint. Exiting code"
               exit 1
             fi

.github/workflows/util/execute_and_retry.sh

@@ -3,28 +3,34 @@
 # This function is for retrying commands in the case they fail. It accepts three arguments
 # $1: Number of retries it will attempt
 # $2: Command to execute
-# $3: (Optional) Command for cleaning up resources if $2 fails
+# $3: (Optional) Command for cleaning up resources if $2 fails.
+# $4: (Optional) Sleep time between run. Default value is 10 seconds
 execute_and_retry () {
   # Warning: The variables called in this function are not local and will be shared with the calling function.
   # Make sure that the variable names do not conflict
   execute_retry_counter=0
   max_execute_retry=$1
   command=$2
   cleanup=$3
+  sleep_time=$4
+  echo "Initiating execute_and_retry.sh script for command $command"
   while [ $execute_retry_counter -lt $max_execute_retry ]; do
+   echo "Attempt Number $execute_retry_counter for execute_and_retry.sh"
    attempt_failed=0
    eval "$command" || attempt_failed=$?
 
    if [ $attempt_failed -ne 0 ]; then
+     echo "Command failed for execute_and_retry.sh, executing cleanup command for another attempt"
      eval "$cleanup"
      execute_retry_counter=$(($execute_retry_counter+1))
-     sleep 5
+     sleep "${sleep_time:-10}"
    else
+     echo "Command executed successfully for execute_and_retry.sh, exiting script"
      break
    fi
 
-   if [ $execute_retry_counter -eq $max_execute_retry ]; then
-     echo "Max retry reached, command failed to execute properly. Exiting code"
+   if [ "$execute_retry_counter" -ge "$max_execute_retry" ]; then
+     echo "Max retry reached, command failed to execute properly. Exiting execute_and_retry.sh script"
      exit 1
    fi
   done