Add python EC2 E2E tests.

sample-apps/python/django_frontend_service/requirements.txt

@@ -1,7 +1,6 @@
 Django~=4.2.9
 requests~=2.31.0
 boto3~=1.34.3
-requests~=2.31.0
 schedule~=1.2.1
-python-dotenv~=1.0.0
+python-dotenv~=1.0.1
 opentelemetry-api~=1.22.0

terraform/python/ec2/main.tf

@@ -0,0 +1,231 @@
+# ------------------------------------------------------------------------
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+# -------------------------------------------------------------------------
+
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}
+
+# Define the provider for AWS
+provider "aws" {}
+
+resource "aws_default_vpc" "default" {}
+
+resource "tls_private_key" "ssh_key" {
+  algorithm = "RSA"
+  rsa_bits = 4096
+}
+
+resource "aws_key_pair" "aws_ssh_key" {
+  key_name = "instance_key-${var.test_id}"
+  public_key = tls_private_key.ssh_key.public_key_openssh
+}
+
+locals {
+  ssh_key_name        = aws_key_pair.aws_ssh_key.key_name
+  private_key_content = tls_private_key.ssh_key.private_key_pem
+}
+
+data "aws_ami" "ami" {
+  owners = ["amazon"]
+  most_recent      = true
+  filter {
+   name = "name"
+   values = ["al2023-ami-2023.3.20240117.0-kernel-6.1-x86_64"]
+  }
+  filter {
+    name   = "state"
+    values = ["available"]
+  }
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
+  filter {
+    name   = "image-type"
+    values = ["machine"]
+  }
+
+  filter {
+    name   = "root-device-name"
+    values = ["/dev/xvda"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+}
+
+resource "aws_instance" "main_service_instance" {
+  ami                                   = data.aws_ami.ami.id # Amazon Linux 2 (free tier)
+  instance_type                         = "t3.small"
+  key_name                              = local.ssh_key_name
+  iam_instance_profile                  = "APP_SIGNALS_EC2_TEST_ROLE"
+  vpc_security_group_ids                = [aws_default_vpc.default.default_security_group_id]
+  associate_public_ip_address           = true
+  instance_initiated_shutdown_behavior  = "terminate"
+  metadata_options {
+    http_tokens = "required"
+  }
+
+  tags = {
+    Name = "main-service-${var.test_id}"
+  }
+}
+
+resource "null_resource" "main_service_setup" {
+  connection {
+    type = "ssh"
+    user = var.user
+    private_key = local.private_key_content
+    host = aws_instance.main_service_instance.public_ip
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      # Install Python and wget
+      "sudo yum install wget -y",
+      "sudo yum install unzip -y",
+      "sudo dnf install -y python3.9",
+      "sudo dnf install -y python3.9-pip",
+
+      # Copy in CW Agent configuration
+      "agent_config='${replace(replace(file("../../ec2/amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'",
+      "echo $agent_config > amazon-cloudwatch-agent.json",
+
+      # Get and run CW agent rpm
+      "wget -O cw-agent.rpm https://amazoncloudwatch-agent-us-east-1.s3.us-east-1.amazonaws.com/amazon_linux/amd64/1.300031.0b313/amazon-cloudwatch-agent.rpm",
+      "sudo rpm -U ./cw-agent.rpm",
+      "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json",
+
+      # Get ADOT Wheel and install it
+      "${var.get_adot_wheel_command}",
+
+      # Get and run the sample application with configuration
+      "aws s3 cp ${var.sample_app_zip} ./python-sample-app.zip",
+      "unzip -o python-sample-app.zip",
+
+      # Export environment variables for instrumentation
+      "cd ./django_frontend_service",
+      "python3.9 -m pip install -r requirements.txt",
+      "export DJANGO_SETTINGS_MODULE=\"django_frontend_service.settings\"",
+      "export OTEL_PYTHON_DISTRO=\"aws_distro\"",
+      "export OTEL_PYTHON_CONFIGURATOR=\"aws_configurator\"",
+      "export OTEL_METRICS_EXPORTER=none",
+      "export OTEL_TRACES_EXPORTER=otlp",
+      "export OTEL_AWS_APP_SIGNALS_ENABLED=true",
+      "export OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4315",
+      "export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4315",
+      "export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc",
+      "export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc",
+      "export OTEL_SERVICE_NAME=python-sample-application-${var.test_id}",
+      "export OTEL_RESOURCE_ATTRIBUTES=aws.hostedin.environment=EC2",
+      "export OTEL_TRACES_SAMPLER=always_on",
+      "python3.9 manage.py migrate",
+      "nohup opentelemetry-instrument python3.9 manage.py runserver 0.0.0.0:8000 --noreload &",
+
+      # The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
+      "sleep 30"
+    ]
+  }
+
+  depends_on = [aws_instance.main_service_instance]
+}
+
+resource "aws_instance" "remote_service_instance" {
+  ami                                   = data.aws_ami.ami.id # Amazon Linux 2 (free tier)
+  instance_type                         = "t3.small"
+  key_name                              = local.ssh_key_name
+  iam_instance_profile                  = "APP_SIGNALS_EC2_TEST_ROLE"
+  vpc_security_group_ids                = [aws_default_vpc.default.default_security_group_id]
+  associate_public_ip_address           = true
+  instance_initiated_shutdown_behavior  = "terminate"
+  metadata_options {
+    http_tokens = "required"
+  }
+
+  tags = {
+    Name = "remote-service-${var.test_id}"
+  }
+}
+
+resource "null_resource" "remote_service_setup" {
+  connection {
+    type = "ssh"
+    user = var.user
+    private_key = local.private_key_content
+    host = aws_instance.remote_service_instance.public_ip
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      # Install Python and wget
+      "sudo yum install wget -y",
+      "sudo yum install unzip -y",
+      "sudo dnf install -y python3.9",
+      "sudo dnf install -y python3.9-pip",
+
+      # Copy in CW Agent configuration
+      "agent_config='${replace(replace(file("../../ec2/amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'",
+      "echo $agent_config > amazon-cloudwatch-agent.json",
+
+      # Get and run CW agent rpm
+      "wget -O cw-agent.rpm https://amazoncloudwatch-agent-us-east-1.s3.us-east-1.amazonaws.com/amazon_linux/amd64/1.300031.0b313/amazon-cloudwatch-agent.rpm",
+      "sudo rpm -U ./cw-agent.rpm",
+      "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json",
+
+      # Get ADOT Wheel and install it
+      "${var.get_adot_wheel_command}",
+
+      # Get and run the sample application with configuration
+      "aws s3 cp ${var.sample_app_zip} ./python-sample-app.zip",
+      "unzip -o python-sample-app.zip",
+
+      # Export environment variables for instrumentation
+      "cd ./django_remote_service",
+      "export DJANGO_SETTINGS_MODULE=\"django_remote_service.settings\"",
+      "python3.9 -m pip install -r requirements.txt --force-reinstall",
+      "export OTEL_PYTHON_DISTRO=\"aws_distro\"",
+      "export OTEL_PYTHON_CONFIGURATOR=\"aws_configurator\"",
+      "export OTEL_METRICS_EXPORTER=none",
+      "export OTEL_TRACES_EXPORTER=otlp",
+      "export OTEL_AWS_APP_SIGNALS_ENABLED=true",
+      "export OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4315",
+      "export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4315",
+      "export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc",
+      "export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc",
+      "export OTEL_SERVICE_NAME=python-sample-remote-application-${var.test_id}",
+      "export OTEL_RESOURCE_ATTRIBUTES=aws.hostedin.environment=EC2",
+      "export OTEL_TRACES_SAMPLER=always_on",
+      "python3.9 manage.py migrate",
+      "nohup opentelemetry-instrument python3.9 manage.py runserver 0.0.0.0:8001 --noreload &",
+
+
+      # The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
+      "sleep 30"
+    ]
+  }
+
+  depends_on = [aws_instance.remote_service_instance]
+}

terraform/python/ec2/output.tf

@@ -0,0 +1,26 @@
+# ------------------------------------------------------------------------
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+# -------------------------------------------------------------------------
+
+output "sample_app_main_service_public_dns" {
+  value = aws_instance.main_service_instance.public_dns
+}
+
+output "sample_app_remote_service_public_ip" {
+  value = aws_instance.remote_service_instance.public_ip
+}
+
+output "ec2_instance_ami" {
+  value = data.aws_ami.ami.id
+}

terraform/python/ec2/variables.tf

@@ -0,0 +1,38 @@
+# ------------------------------------------------------------------------
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+# -------------------------------------------------------------------------
+
+variable "test_id" {
+  default = "dummy-123"
+}
+
+variable "aws_region" {
+  default = "<aws-region>"
+}
+
+variable "user" {
+  default = "ec2-user"
+}
+
+variable "sample_app_zip" {
+  default = "s3://<bucket-name>/<zip>"
+}
+
+variable "get_adot_wheel_command" {
+  default = "aws s3 cp s3://<bucket-name>/<whl> ./<whl> && pip install <whl>"
+}
+
+variable "get_cw_agent_rpm_command" {
+  default = "<command> s3://<bucket-name>/<jar>"
+}

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -39,7 +39,7 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   EKS_CLIENT_CALL_METRIC("/expected-data-template/eks/client-call-metric.mustache"),
   EKS_CLIENT_CALL_TRACE("/expected-data-template/eks/client-call-trace.mustache"),
 
-  /** EC2 Test Case Validations */
+  /** Java EC2 Test Case Validations */
   EC2_OUTGOING_HTTP_CALL_LOG("/expected-data-template/ec2/outgoing-http-call-log.mustache"),
   EC2_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/ec2/outgoing-http-call-metric.mustache"),
   EC2_OUTGOING_HTTP_CALL_TRACE("/expected-data-template/ec2/outgoing-http-call-trace.mustache"),
@@ -55,6 +55,23 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   EC2_CLIENT_CALL_LOG("/expected-data-template/ec2/client-call-log.mustache"),
   EC2_CLIENT_CALL_METRIC("/expected-data-template/ec2/client-call-metric.mustache"),
   EC2_CLIENT_CALL_TRACE("/expected-data-template/ec2/client-call-trace.mustache"),
+
+  /** Python EC2 Test Case Validations */
+  PYTHON_EC2_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/ec2/outgoing-http-call-log.mustache"),
+  PYTHON_EC2_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/ec2/outgoing-http-call-metric.mustache"),
+  PYTHON_EC2_OUTGOING_HTTP_CALL_TRACE("/expected-data-template/python/ec2/outgoing-http-call-trace.mustache"),
+
+  PYTHON_EC2_AWS_SDK_CALL_LOG("/expected-data-template/python/ec2/aws-sdk-call-log.mustache"),
+  PYTHON_EC2_AWS_SDK_CALL_METRIC("/expected-data-template/python/ec2/aws-sdk-call-metric.mustache"),
+  PYTHON_EC2_AWS_SDK_CALL_TRACE("/expected-data-template/python/ec2/aws-sdk-call-trace.mustache"),
+
+  PYTHON_EC2_REMOTE_SERVICE_LOG("/expected-data-template/python/ec2/remote-service-log.mustache"),
+  PYTHON_EC2_REMOTE_SERVICE_METRIC("/expected-data-template/python/ec2/remote-service-metric.mustache"),
+  PYTHON_EC2_REMOTE_SERVICE_TRACE("/expected-data-template/python/ec2/remote-service-trace.mustache"),
+
+  PYTHON_EC2_CLIENT_CALL_LOG("/expected-data-template/python/ec2/client-call-log.mustache"),
+  PYTHON_EC2_CLIENT_CALL_METRIC("/expected-data-template/python/ec2/client-call-metric.mustache"),
+  PYTHON_EC2_CLIENT_CALL_TRACE("/expected-data-template/python/ec2/client-call-trace.mustache"),
   ;
 
   private String path;

validator/src/main/resources/expected-data-template/python/ec2/aws-sdk-call-log.mustache

@@ -0,0 +1,17 @@
+[{
+  "HostedIn.Environment": "^EC2$",
+  "Operation": "GET aws-sdk-call",
+  "OTelLib": "^AwsSpanMetricsProcessor$",
+  "Version": "^1$",
+  "aws.span.kind": "^LOCAL_ROOT$"
+},
+{
+  "HostedIn.Environment": "^EC2$",
+  "Operation": "GET aws-sdk-call",
+  "OTelLib": "^AwsSpanMetricsProcessor$",
+  "Version": "^1$",
+  "RemoteService": "AWS.SDK.S3",
+  "RemoteOperation": "GetBucketLocation",
+  "RemoteTarget": "::s3:::e2e-test-bucket-name",
+  "aws.span.kind": "^CLIENT$"
+}]