Check if remote application is deployed properly

Check if ec2 instance inline code fail

Author: harrryr

.github/workflows/actions/execute_and_retry/action.yml

@@ -0,0 +1,71 @@
+# Reusuable Action for executing commands and retrying them if it fails
+name: Command Retry Logic
+
+inputs:
+  pre-command:
+    required: false
+    type: string
+  max_retry:
+    required: false
+    type: number
+    default: 2
+  command:
+    required: true
+    type: string
+  cleanup:
+    required: false
+    type: string
+  post-command:
+    required: false
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Run pre-command
+      shell: bash
+      env:
+        PRE_COMMAND: ${{ inputs.pre-command }}
+      run: |
+        $PRE_COMMAND
+
+    - name: Run command
+      shell: bash
+      env:
+        MAX_RETRY: ${{ inputs.max_retry }}
+        COMMAND: ${{ inputs.command }}
+        CLEANUP: ${{ inputs.cleanup }}
+      run: |
+        retry_counter=0
+        while [ $retry_counter -lt $MAX_RETRY ]; do
+        attempt_failed=0
+        eval "$COMMAND" || attempt_failed=$?
+
+        if [ $attempt_failed -ne 0 ]; then
+          eval "$CLEANUP"
+          retry_counter=$(($retry_counter+1))
+          sleep 5
+        else
+          break
+        fi
+
+        if [ $retry_counter -eq $max_retry ]; then
+          echo "Max retry reached, command failed to execute properly. Exiting code"
+          exit 1
+        fi
+        done
+
+    - name: Run post command
+      shell: bash
+      env:
+        POST_COMMAND: ${{ inputs.post-command }}
+      run: $POST_COMMAND
+
+
+
+
+
+
+
+
+

.github/workflows/appsignals-e2e-ec2-test.yml

@@ -31,6 +31,7 @@ env:
   LOG_GROUP_NAME: /aws/appsignals/generic
   TEST: ${{ inputs.test }}
   GET_ADOT_JAR_COMMAND: "wget -O adot.jar https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest/download/aws-opentelemetry-agent.jar"
+  GET_CW_AGENT_RPM_COMMAND: "wget -O cw-agent.rpm https://amazoncloudwatch-agent-${{ inputs.aws-region }}.s3.${{ inputs.aws-region }}.amazonaws.com/amazon_linux/amd64/1.300031.0b313/amazon-cloudwatch-agent.rpm"
   TEST_RESOURCES_FOLDER: /home/runner/work/aws-application-signals-test-framework/aws-application-signals-test-framework
 
 
@@ -42,9 +43,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set CW Agent RPM environment variable
-        run: echo GET_CW_AGENT_RPM_COMMAND="wget -O cw-agent.rpm https://amazoncloudwatch-agent-${{ inputs.aws-region }}.s3.${{ inputs.aws-region }}.amazonaws.com/amazon_linux/amd64/1.300031.0b313/amazon-cloudwatch-agent.rpm" >> $GITHUB_ENV
-
       - name: Generate testing id
         run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV
 
@@ -67,17 +65,17 @@ jobs:
           aws-region: ${{ inputs.aws-region }}
 
       - name: Set up terraform
-        run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          execute_and_retry 2 "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg"   
-          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
-          sudo apt update && sudo apt install terraform
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg"
+          post-command: 'echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list \
+          sudo apt update && sudo apt install terraform'
 
       - name: Initiate Terraform
-        working-directory: terraform/ec2
-        run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          execute_and_retry 2 "terraform init && terraform validate" "rm -rf .terraform && rm -rf .terraform.lock.hcl"
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "cd ${{ env.TEST_RESOURCES_FOLDER }}/terraform/ec2 && terraform init && terraform validate"
+          cleanup: "rm -rf .terraform && rm -rf .terraform.lock.hcl"
 
       - name: Deploy sample app via terraform and wait for endpoint to come online
         working-directory: terraform/ec2
@@ -108,10 +106,26 @@ jobs:
             # Attempts to connect will be made for up to 10 minutes
             if [ $deployment_failed -eq 0 ]; then
               echo "Attempting to connect to the endpoint"
-              sample_app_endpoint=http://$(terraform output sample_app_main_service_public_dns):8080
+              main_sample_app_endpoint=http://$(terraform output sample_app_main_service_public_dns):8080
+              attempt_counter=0
+              max_attempts=30
+              until $(curl --output /dev/null --silent --head --fail $(echo "$main_sample_app_endpoint" | tr -d '"')); do
+                if [ ${attempt_counter} -eq ${max_attempts} ];then
+                  echo "Failed to connect to endpoint. Will attempt to redeploy sample app."
+                  deployment_failed=1
+                  break
+                fi
+          
+                printf '.'
+                attempt_counter=$(($attempt_counter+1))
+                sleep 10
+              done
+          
+              echo "Attempting to connect to the remote sample app endpoint"
+              remote_sample_app_endpoint=http://$(terraform output sample_app_remote_service_public_ip):8080/healthcheck
               attempt_counter=0
-              max_attempts=60
-              until $(curl --output /dev/null --silent --head --fail $(echo "$sample_app_endpoint" | tr -d '"')); do
+              max_attempts=30
+              until $(curl --output /dev/null --silent --head --fail $(echo "$remote_sample_app_endpoint" | tr -d '"')); do
                 if [ ${attempt_counter} -eq ${max_attempts} ];then
                   echo "Failed to connect to endpoint. Will attempt to redeploy sample app."
                   deployment_failed=1
@@ -164,9 +178,9 @@ jobs:
           curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call/
           
       - name: Build Gradlew
-        run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          execute_and_retry 2 ./gradlew
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "./gradlew"
 
       # Validation for pulse telemetry data
       - name: Validate generated EMF logs

.github/workflows/appsignals-e2e-eks-test.yml

@@ -94,38 +94,42 @@ jobs:
       - name: Set up kubeconfig
         run: aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}
 
+      - name: Download eksctl
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          pre-command: 'mkdir ${{ github.workspace }}/eksctl'
+          command: 'curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz"'
+
       - name: Install eksctl
         run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          mkdir ${{ github.workspace }}/eksctl
-          curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz"
-          execute_and_retry 2 "tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz"
+          tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz
           echo "${{ github.workspace }}/eksctl" >> $GITHUB_PATH
 
       - name: Create role for AWS access from the sample app
         id: create_service_account
-        run: |
-          eksctl create iamserviceaccount \
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "eksctl create iamserviceaccount \
           --name service-account-${{ env.TESTING_ID }} \
           --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \
           --cluster ${{ inputs.test-cluster-name }} \
           --role-name eks-s3-access-${{ env.TESTING_ID }} \
           --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \
           --region ${{ inputs.aws-region }} \
-          --approve
+          --approve"
 
       - name: Set up terraform
-        run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          execute_and_retry 2 "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg"          
-          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
-          sudo apt update && sudo apt install terraform
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg"
+          post-command: 'echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list \
+          sudo apt update && sudo apt install terraform'
 
       - name: Initiate Terraform
-        working-directory: terraform/eks
-        run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          execute_and_retry 2 "terraform init && terraform validate" "rm -rf .terraform && rm -rf .terraform.lock.hcl"
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "cd ${{ env.TEST_RESOURCES_FOLDER }}/terraform/eks && terraform init && terraform validate"
+          cleanup: "rm -rf .terraform && rm -rf .terraform.lock.hcl"
 
       - name: Deploy sample app via terraform and wait for the endpoint to come online
         id: deploy-sample-app
@@ -159,10 +163,25 @@ jobs:
             # If the deployment_failed is still 0, then the terraform deployment succeeded and now try to connect to the endpoint 
             # after installing App Signals. Attempts to connect will be made for up to 10 minutes
             if [ $deployment_failed -eq 0 ]; then
+              source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
+              execute_and_retry 2 \
+              "${{ env.TEST_RESOURCES_FOLDER }}/enablement-script/scripts/eks/appsignals/enable-app-signals.sh \
+              ${{ inputs.test-cluster-name }} \
+              ${{ inputs.aws-region }} \
+              ${{ env.SAMPLE_APP_NAMESPACE }}" \
+              "${{ env.TEST_RESOURCES_FOLDER }}/enablement-script/scripts/eks/appsignals/clean-app-signals.sh \
+              ${{ inputs.test-cluster-name }} \
+              ${{ inputs.aws-region }} \
+              ${{ env.SAMPLE_APP_NAMESPACE }} && \
+              aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}"
+          
+              kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }}
+              kubectl wait --for=condition=Ready pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }}
+          
               echo "Attempting to connect to the main sample app endpoint"
               main_sample_app_endpoint=http://$(terraform output sample_app_endpoint)
               attempt_counter=0
-              max_attempts=60
+              max_attempts=30
               until $(curl --output /dev/null --silent --head --fail $(echo "$main_sample_app_endpoint" | tr -d '"')); do
                 if [ ${attempt_counter} -eq ${max_attempts} ];then
                   echo "Failed to connect to endpoint. Will attempt to redeploy sample app."
@@ -176,12 +195,10 @@ jobs:
               done
           
               echo "Attempting to connect to the remote sample app endpoint"
-              remote_service_pod_ip=$(kubectl get pods -n ${{ env.SAMPLE_APP_NAMESPACE }} --selector=app=remote-app -o jsonpath='{.items[0].status.podIP}')
-              remote_sample_app_endpoint=http://$remote_service_pod_ip/healthcheck
-              echo $remote_service_pod_ip
+              remote_sample_app_endpoint=http://$(terraform output sample_remote_app_endpoint)/healthcheck
               echo $remote_sample_app_endpoint
               attempt_counter=0
-              max_attempts=60
+              max_attempts=30
               until $(curl --output /dev/null --silent --head --fail $(echo "$remote_sample_app_endpoint" | tr -d '"')); do
                 if [ ${attempt_counter} -eq ${max_attempts} ];then
                   echo "Failed to connect to endpoint. Will attempt to redeploy sample app."
@@ -198,6 +215,15 @@ jobs:
             # If the deployment_failed is 1 then either the terraform deployment or the endpoint connection failed, so first destroy the
             # resources created from terraform and try again.
             if [ $deployment_failed -eq 1 ]; then
+              echo "Cleaning up App Signal"
+              ./clean-app-signals.sh \
+              ${{ inputs.test-cluster-name }} \
+              ${{ inputs.aws-region }} \
+              ${{ env.SAMPLE_APP_NAMESPACE }}
+          
+              # Running clean-app-signal.sh removes the current cluster from the config. Update the cluster again for subsequent runs.
+              aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}
+            
               echo "Destroying terraform"
               terraform destroy -auto-approve \
                 -var="test_id=${{ env.TESTING_ID }}" \
@@ -220,24 +246,6 @@ jobs:
             fi
           done
 
-      - name: Install AppSignals addon
-        run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          execute_and_retry 2 \
-          "${GITHUB_WORKSPACE}/enablement-script/scripts/eks/appsignals/enable-app-signals.sh \
-          ${{ inputs.test-cluster-name }} \
-          ${{ inputs.aws-region }} \
-          ${{ env.SAMPLE_APP_NAMESPACE }}" \
-          " echo "Cleaning up App Signal" \
-          ./clean-app-signals.sh \
-          ${{ inputs.test-cluster-name }} \
-          ${{ inputs.aws-region }} \
-          ${{ env.SAMPLE_APP_NAMESPACE }} && \
-          aws eks update-kubeconfig --name ${{ inputs.test-cluster-name }} --region ${{ inputs.aws-region }}"
-          
-          kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }}
-          kubectl wait --for=condition=Ready pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }}
-
       - name: Get remote service pod name and IP
         run: |
           echo "REMOTE_SERVICE_DEPLOYMENT_NAME=$(kubectl get deployments -n ${{ env.SAMPLE_APP_NAMESPACE }} --selector=app=remote-app -o jsonpath='{.items[0].metadata.name}')" >> $GITHUB_ENV
@@ -267,9 +275,9 @@ jobs:
           curl -S -s http://${{ env.APP_ENDPOINT }}/client-call/
 
       - name: Build Gradlew
-        run: |
-          source ${{ env.TEST_RESOURCES_FOLDER }}/.github/workflows/util/execute_and_retry.sh
-          execute_and_retry 2 "./gradlew"
+        uses: ./.github/workflows/actions/execute_and_retry
+        with:
+          command: "./gradlew"
 
       # Validation for app signals telemetry data
       - name: Call endpoint and validate generated EMF logs

.github/workflows/test.yml

@@ -104,6 +104,8 @@ resource "null_resource" "main_service_setup" {
 
   provisioner "remote-exec" {
     inline = [
+      # Make the Terraform fail if any step throws an error
+      "set -o errexit",
       # Install Java 11 and wget
       "sudo yum install wget java-11-amazon-corretto -y",
 
@@ -165,6 +167,8 @@ resource "null_resource" "remote_service_setup" {
 
   provisioner "remote-exec" {
     inline = [
+      # Make the Terraform fail if any step throws an error
+      "set -o errexit",
       # Install Java 11 and wget
       "sudo yum install wget java-11-amazon-corretto -y",