feat: assert deployment success

Author: andhreljaKern

.github/workflows/k8s_apply.yml

@@ -26,6 +26,12 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
+    - name: GitHub Configuration
+      run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+
+    - name: Clone cicd-deployment-scripts
+      run: git clone https://github.com/code-kern-ai/cicd-deployment-scripts.git
+
     - name: Azure Cloud Login
       uses: azure/login@v2
       with:
@@ -57,9 +63,18 @@ jobs:
 
     - name: Apply Kustomization
       shell: bash
-      # env:
-      #   DRY_RUN: ${{ github.event_name == 'push' && 'none' || 'client' }}
       run: |
         kubectl apply \
           --filename apply.yml \
           --namespace ${{ env.KUBERNETES_NAMESPACE }}
+
+    - name: Assert Deploy Success
+      shell: bash
+      run: |
+        bash cicd-deployment-scripts/k8s/assert_deploy_success.sh \
+          -n ${{ env.KUBERNETES_NAMESPACE }} \
+          -p ${{ github.event.pull_request.number }}
+
+    - name: Revert on failure
+      if: failure()
+      run: echo "::error::Deployment failed, revert this PR" && exit 1

k8s/assert_deploy_success.sh

@@ -2,39 +2,41 @@
 set -e
 
 KUBERNETES_NAMESPACE=""
-KUBERNETES_DEPLOYMENT_NAME=""
-APPLICATION_STARTUP_MESSAGE="Application startup complete"
+PR_NUMBER=""
+DEPLOY_SUCCESSFUL=true
 
-while getopts n:d:m: flag
+while getopts n:p: flag
 do
     case "${flag}" in
         n) KUBERNETES_NAMESPACE=${OPTARG};;
-        d) KUBERNETES_DEPLOYMENT_NAME=${OPTARG};;
-        m) APPLICATION_STARTUP_MESSAGE=${OPTARG};;
+        p) PR_NUMBER=${OPTARG};;
     esac
 done
 
 kubectl config set-context --current --namespace=$KUBERNETES_NAMESPACE
 echo "Context set to namespace: \"$KUBERNETES_NAMESPACE\""
 
-echo "Reading logs to determine application startup status for '$KUBERNETES_DEPLOYMENT_NAME'"
-echo "Searching for message: '$APPLICATION_STARTUP_MESSAGE'"
+UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
+while IFS= read -r file; do
+    if [[ $file != apps/* ]]; then
+        continue
+    fi
 
-LOG_CONTENTS=$(kubectl logs deployment/${KUBERNETES_DEPLOYMENT_NAME} \
-    || echo "Waiting for application startuop ...")
+    deploy=$(echo "$file" | cut -d/ -f 2)
 
-while [[ "$LOG_CONTENTS" != *"$APPLICATION_STARTUP_MESSAGE"* ]]; do
-    echo "Waiting for application startup..."
-    sleep 3
-    LOG_CONTENTS=$(kubectl logs deployment/${KUBERNETES_DEPLOYMENT_NAME} \
-        || echo "Waiting for application startuop ...")
-    
-    if [[ "$LOG_CONTENTS" == *"ERROR"* ]]; then
-        echo "Application startup failed:"
-        echo "$LOG_CONTENTS"
-        exit 1
+    set +e
+    kubectl rollout status deploy $deploy --timeout 30s
+    if [ $? -ne 0 ]; then
+        echo "::error::Deployment $deploy failed to rollout"
+        DEPLOY_SUCCESSFUL=false
     fi
-done
+    set -e
+    
+done <<< "$UPDATED_FILES"
 
-echo "Application startup successful:"
-echo "$LOG_CONTENTS"
+if [ $DEPLOY_SUCCESSFUL = true ]; then
+    echo "::notice::Automated release successful"
+else
+    echo "::error::Automated release failed"
+    exit 1
+fi