Merge branch 'main' into remove-public-endpoint-k8s

Author: harrryr

.github/workflows/traffic-generator-image-build.yml

@@ -0,0 +1,100 @@
+# This workflow will build and push the traffic generator to each region whenever there is an update made to the traffic-generator folder.
+# This image will be used by EKS and K8s test to call sample app endpoints while the zip files will be used by EC2 Platforms
+name: Create and Push Traffic Generator
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'sample-apps/traffic-generator/**'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
+  E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        aws-region: ['af-south-1','ap-east-1','ap-northeast-1','ap-northeast-2','ap-northeast-3','ap-south-1','ap-south-2','ap-southeast-1',
+                     'ap-southeast-2','ap-southeast-3','ap-southeast-4','ca-central-1','eu-central-1','eu-central-2','eu-north-1',
+                     'eu-south-1','eu-south-2','eu-west-1','eu-west-2','eu-west-3','il-central-1','me-central-1','me-south-1', 'sa-east-1',
+                     'us-east-1','us-east-2', 'us-west-1', 'us-west-2']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ env.E2E_TEST_ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }}
+          aws-region: us-east-1
+
+      - name: Retrieve account
+        uses: aws-actions/aws-secretsmanager-get-secrets@v1
+        with:
+          secret-ids: |
+            ACCOUNT_ID, region-account/${{ matrix.aws-region }}
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ env.ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }}
+          aws-region: ${{ matrix.aws-region }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build, tag, and push image to Amazon ECR
+        working-directory: sample-apps/traffic-generator
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: e2e-test-resource
+          IMAGE_TAG: traffic-generator
+        run: |
+          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+
+  upload-files-to-s3:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        aws-region: ['af-south-1','ap-east-1','ap-northeast-1','ap-northeast-2','ap-northeast-3','ap-south-1','ap-south-2','ap-southeast-1',
+                     'ap-southeast-2','ap-southeast-3','ap-southeast-4','ca-central-1','eu-central-1','eu-central-2','eu-north-1',
+                     'eu-south-1','eu-south-2','eu-west-1','eu-west-2','eu-west-3','il-central-1','me-central-1','me-south-1', 'sa-east-1',
+                     'us-east-1','us-east-2', 'us-west-1', 'us-west-2']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ env.E2E_TEST_ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }}
+          aws-region: us-east-1
+
+      - name: Retrieve account
+        uses: aws-actions/aws-secretsmanager-get-secrets@v1
+        with:
+          secret-ids: |
+            ACCOUNT_ID, region-account/${{ matrix.aws-region }}
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ env.ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }}
+          aws-region: ${{ matrix.aws-region }}
+
+      - name: Upload traffic generator files
+        working-directory: sample-apps/traffic-generator
+        run: |
+          zip traffic-generator.zip ./index.js ./package.json
+          aws s3 cp traffic-generator.zip s3://aws-appsignals-sample-app-prod-${{ matrix.aws-region }}/traffic-generator.zip

.github/workflows/util/clean/ec2_instance_cleanup/cleaner.py

@@ -15,17 +15,74 @@
 # Create an EC2 client
 session = boto3.Session()
 ec2 = session.client('ec2')
+autoscaling = session.client('autoscaling')
 
 # configure logging
 logging.basicConfig(level=logging.INFO)
 
+def _get_autoscaling_groups_to_delete():
+    logging.info("Start scanning autoscaling group...")
+
+    current_time = datetime.now(timezone.utc)
+    time_threshold = current_time - timedelta(hours=3)
+    groups_to_delete = []
+
+     # Initialize the paginator
+    paginator = autoscaling.get_paginator('describe_auto_scaling_groups')
+
+    # Iterate through each page of results
+    for page in paginator.paginate():
+        auto_scaling_groups = page['AutoScalingGroups']
+        for asg in auto_scaling_groups:
+            asg_name = asg['AutoScalingGroupName']
+            tags = asg['Tags']
+
+            eks_tag_present = any(tag['Key'] == 'eks:cluster-name' for tag in tags)
+            if eks_tag_present:
+                logging.info(f"Skipping autoscaling group with 'eks:cluster-name' tag: {asg_name}.")
+                continue
+
+            if not _is_active(asg):
+                logging.info(f"Skipping autoscaling group {asg_name} with terminating instances.")
+                continue
+
+            logging.info(f"autoscaling group {asg_name} is active.")
+
+            creation_time = asg['CreatedTime']
+            if creation_time < time_threshold:
+                print(f"Autoscaling group: {asg_name} will be deleted.")
+                groups_to_delete.append(asg)
+    
+    logging.info(f"{len(groups_to_delete)} autoscaling groups are active for more than 3 hours.")
+
+    return groups_to_delete
+
+
+def _delete_autoscaling_groups(auto_scaling_groups):
+    for asg in auto_scaling_groups:
+        try:
+            asg_name = asg['AutoScalingGroupName']
+            response = autoscaling.delete_auto_scaling_group(AutoScalingGroupName=asg_name, ForceDelete=True)
+            logging.info("===== Response for delete autoscaling group request =====")
+            logging.info(response)
+        except Exception as e:
+            logging.info(f"Error terminating instances: {e}")
+
+def _is_active(asg):
+    for instance in asg['Instances']:
+        if instance['LifecycleState'] in [
+            'Terminating', 'Terminating:Wait', 'Terminating:Proceed'
+        ]:
+            return False
+    return True
+
 
 def _get_instances_to_terminate():
     # Get all the running instances
-    logging.info("Getting all running instances")
+    logging.info("Start scanning instances")
     running_filter = [{'Name': 'instance-state-name', 'Values': [INSTANCE_STATE_RUNNING]}]
     running_instances = _get_all_instances_by_filter(filters=running_filter)
-    logging.info(f"Currently {len(running_instances)} are running.")
+    logging.info(f"{len(running_instances)} instances are running.")
 
     # Filter instances that have been running for more than 3 hours
     logging.info("Filtering instances that have been running for more than 3 hours")
@@ -42,10 +99,13 @@ def _get_instances_to_terminate():
     logging.info("Filtering instances that should not be terminated based on conditions")
     instances_to_terminate = []
     for instance in instances_running_more_than_3hrs:
-        if (not _is_eks_cluster_instance(instance)
-                and not _is_k8s_cluster_instance(instance)
-                and not _is_tagged_do_not_delete(instance)):
-            instances_to_terminate.append(instance)
+        if (not _is_k8s_cluster_instance(instance) and not _is_tagged_do_not_delete(instance)):
+            group_name = _get_associated_autoscaling_group_name(instance)
+            if group_name != None:
+                logging.info(f"Instance {instance['InstanceId']} is associated with autoscaling group {group_name}, skip the termination.")
+            else:
+                instances_to_terminate.append(instance)
+
     logging.info(f"{len(instances_to_terminate)} instances will be terminated.")
 
     return instances_to_terminate
@@ -70,13 +130,6 @@ def _get_all_instances_by_filter(filters: List[dict]):
     return filtered_instances
 
 
-def _is_eks_cluster_instance(instance):
-    security_groups = instance.get('SecurityGroups', [])
-    if any(group['GroupName'].startswith(EKS_CLUSTER_SECURITY_GROUP_PREFIX) for group in security_groups):
-        return True
-    return False
-
-
 def _is_k8s_cluster_instance(instance):
     tags = instance.get('Tags', [])
     if 'Name' in tags and tags['Name'].startswith(K8S_INSTANCE_NAME_PREFIX):
@@ -92,12 +145,21 @@ def _is_tagged_do_not_delete(instance):
         return True
     return False
 
-
-def _prepare_report_and_upload(instances_to_terminate) -> bool:
-    json_data = json.dumps(instances_to_terminate, default=str)
+def _get_associated_autoscaling_group_name(instance):
+    tags = instance.get('Tags', [])
+    asg_tag = next((tag for tag in tags if tag['Key'] == 'aws:autoscaling:groupName'), None)
+    if asg_tag is None:
+        return None
+    return asg_tag['Value']
+
+def _prepare_report_and_upload(groups_to_delete, instances_to_terminate) -> bool:
+    json_data = json.dumps({
+        "autoscalingGroups": groups_to_delete,
+        "standaloneInstances": instances_to_terminate
+    }, default=str)
     # save as a json file with timestamp
     timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
-    filename = f"report-instances-to-terminate-{timestamp}.json"
+    filename = f"report-resources-to-clean-{timestamp}.json"
     with open(filename, "w") as f:
         f.write(json_data)
 
@@ -116,24 +178,26 @@ def _prepare_report_and_upload(instances_to_terminate) -> bool:
 def _terminate_instances(instances_to_terminate):
     # Terminate the instances
     instance_ids = [instance['InstanceId'] for instance in instances]
-    logging.info("Number of instances terminating: " + str(len(instance_ids)))
     try:
         response = ec2.terminate_instances(InstanceIds=instance_ids)
-        logging.info("===== Response for terminate request =====")
+        logging.info("===== Response for terminate instances request =====")
         logging.info(response)
     except Exception as e:
         logging.info(f"Error terminating instances: {e}")
 
 
 if __name__ == '__main__':
+    groups = _get_autoscaling_groups_to_delete()
     instances = _get_instances_to_terminate()
-    if len(instances) == 0:
-        logging.info("No instances to terminate")
+    
+    if len(groups) == 0 and len(instances) == 0:
+        logging.info("No resource to terminate")
         exit(0)
 
-    report_successful = _prepare_report_and_upload(instances)
+    report_successful = _prepare_report_and_upload(groups, instances)
     if not report_successful:
         logging.error("Failed to prepare report and upload. Aborting termination of instances.")
         exit(1)
 
+    _delete_autoscaling_groups(groups)
     _terminate_instances(instances)

.gitignore

@@ -1,9 +1,7 @@
+.DS_Store
 .idea
 # Ignore Gradle project-specific cache directory
 .gradle
 
 # Ignore Gradle build output directory
 build
-
-# Ignore the resource cleanup reports
-**/report-*.json

sample-apps/traffic-generator/Dockerfile

@@ -0,0 +1,21 @@
+# Use the official lightweight Node.js 16 image.
+# https://hub.docker.com/_/node
+# FROM node:16-slim
+FROM public.ecr.aws/eks-distro-build-tooling/nodejs:16
+
+# Create and change to the app directory
+WORKDIR /usr/src/app
+
+# Copy application dependency manifests to the container image.
+# A wildcard is used to ensure copying both package.json AND package-lock.json (if available).
+# Copying this first prevents re-running npm install on every code change.
+COPY package*.json ./
+
+# Install dependencies
+RUN npm install
+
+# Copy local code to the container image.
+COPY . .
+
+# Run the web service on container startup.
+CMD [ "npm", "start" ]

sample-apps/traffic-generator/index.js

@@ -0,0 +1,73 @@
+const axios = require('axios');
+
+// Send API requests to the sample app
+const sendRequests = async (urls) => {
+    try {
+        const fetchPromises = urls.map(url => axios.get(url));
+        const responses = await Promise.all(fetchPromises);
+
+        // Handle the responses
+        responses.forEach((response, index) => {
+            if (response.status === 200) {
+                const data = response.data;
+                console.log(`Response from ${urls[index]}:`, data);
+            } else {
+                console.error(`Failed to fetch ${urls[index]}:`, response.statusText);
+            }
+        });
+    } catch (error) {
+        console.error('Error sending GET requests:', error);
+    }
+}
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+// This loop will run until the environment variables are available
+const waitForEnvVariables = async () => {
+    while (!process.env.MAIN_ENDPOINT || !process.env.REMOTE_ENDPOINT || !process.env.ID || !process.env.CANARY_TYPE) {
+        console.log('Environment variables not set. Waiting for 10 seconds...');
+        await sleep(10000); // Wait for 10 seconds
+    }
+};
+
+// Traffic generator that sends traffic every specified interval. Send request immediately then every 2 minutes afterwords
+const trafficGenerator = async (interval) => {
+    await waitForEnvVariables();
+
+    const mainEndpoint = process.env.MAIN_ENDPOINT;
+    const remoteEndpoint = process.env.REMOTE_ENDPOINT;
+    const id = process.env.ID;
+    const canaryType = process.env.CANARY_TYPE
+
+    let urls = [
+        `http://${mainEndpoint}/outgoing-http-call`,
+        `http://${mainEndpoint}/aws-sdk-call?ip=${remoteEndpoint}&testingId=${id}`,
+        `http://${mainEndpoint}/remote-service?ip=${remoteEndpoint}&testingId=${id}`,
+        `http://${mainEndpoint}/client-call`
+    ];
+
+    if (canaryType === 'java-eks' || canaryType === 'python-eks') {
+        urls.push(`http://${mainEndpoint}/mysql`)
+    }
+
+    // Need to call some APIs so that it exceeds the metric limiter threshold and make the test
+    // APIs generate AllOtherOperations metric. Sleep for a minute to let cloudwatch service process the API call
+    // Calling it here before calling the remote sample app endpoint because the API generated by it is validated
+    // for AllOtherRemoteOperations in the metric validation step
+    if (canaryType === 'java-metric-limiter'){
+        const fakeUrls = [
+            `http://${mainEndpoint}`,
+            `http://${mainEndpoint}/fake-endpoint`
+        ]
+        // Send the fake requests and wait a minute
+        await sendRequests(fakeUrls);
+        await sleep(60000);
+    }
+
+    await sendRequests(urls);
+    setInterval(() => sendRequests(urls), interval);
+}
+
+const interval = 60 * 1000;
+// Start sending GET requests every minute (60,000 milliseconds)
+trafficGenerator(interval);

sample-apps/traffic-generator/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "traffic-generator",
+  "version": "1.0.0",
+  "description": "A simple traffic generator that sends GET requests to a list of URLs every 2 minutes",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js"
+  },
+  "dependencies": {
+    "axios": "^1.4.0"
+  }
+}

terraform/.gitignore

@@ -0,0 +1,2 @@
+.terraform*
+terraform.tfstate*