Skip to content

Use Application Signals wherever possible #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# https://help.github.com/en/articles/about-code-owners

# Default owners for the entire repo
* @aws-observability/aws-appsignals-team
* @aws-observability/aws-application-signals-team

# Owners for aws-otel-distro folder
/aws-otel-distro/ @zzhlogin @thpierce @srprash @AsakerMohd
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
## SPDX-License-Identifier: Apache-2.0

# This is a reusable workflow for running the Python E2E Canary test for App Signals.
# This is a reusable workflow for running the Python E2E Canary test for Application Signals.
# It is meant to be called from another workflow.
# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
name: App Signals Enablement E2E Testing - Python EC2 Use Case
name: Application Signals Enablement E2E Testing - Python EC2 Use Case
on:
workflow_call:
inputs:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
## SPDX-License-Identifier: Apache-2.0

# This is a reusable workflow for running the E2E test for App Signals.
# This is a reusable workflow for running the E2E test for Application Signals.
# It is meant to be called from another workflow.
# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
name: App Signals Enablement E2E Testing - Python EKS
name: Application Signals Enablement E2E Testing - Python EKS
on:
workflow_call:
inputs:
Expand All @@ -14,10 +14,10 @@ on:
test-cluster-name:
required: true
type: string
appsignals-adot-image:
application-signals-adot-image:
required: false
type: string
appsignals-adot-image-tag:
application-signals-adot-image-tag:
required: false
type: string
caller-workflow-name:
Expand Down Expand Up @@ -164,9 +164,9 @@ jobs:
fi

# If the deployment_failed is still 0, then the terraform deployment succeeded and now try to connect to the endpoint
# after installing App Signals. Attempts to connect will be made for up to 10 minutes
# after installing Application Signals. Attempts to connect will be made for up to 10 minutes
if [ $deployment_failed -eq 0 ]; then
echo "Installing app signals to the sample app"
echo "Installing application signals to the sample app"
source ${GITHUB_WORKSPACE}/.github/workflows/util/execute_and_retry.sh
execute_and_retry 2 \
"${GITHUB_WORKSPACE}/enablement-script/enable-app-signals.sh \
Expand All @@ -182,7 +182,7 @@ jobs:
if [ "${{ inputs.caller-workflow-name }}" == "main-build" ]; then
echo "Patching staging adot image for main build:"
execute_and_retry 2 "kubectl patch deploy -namazon-cloudwatch amazon-cloudwatch-observability-controller-manager --type='json' \
-p='[{"op": \"replace\", \"path\": \"/spec/template/spec/containers/0/args/0\", \"value\": \"--auto-instrumentation-python-image=${{ inputs.appsignals-adot-image }}:${{ inputs.appsignals-adot-image-tag }}\"}]'"
-p='[{"op": \"replace\", \"path\": \"/spec/template/spec/containers/0/args/0\", \"value\": \"--auto-instrumentation-python-image=${{ inputs.application-signals-adot-image }}:${{ inputs.application-signals-adot-image-tag }}\"}]'"
execute_and_retry 2 "kubectl delete pods --all -n amazon-cloudwatch"
execute_and_retry 2 "kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch"
fi
Expand Down Expand Up @@ -210,7 +210,7 @@ jobs:
# If the deployment_failed is 1 then either the terraform deployment or the endpoint connection failed, so first destroy the
# resources created from terraform and try again.
if [ $deployment_failed -eq 1 ]; then
echo "Cleaning up App Signal"
echo "Cleaning up Application Signal"
${GITHUB_WORKSPACE}/enablement-script/clean-app-signals.sh \
${{ inputs.test-cluster-name }} \
${{ inputs.aws-region }} \
Expand Down Expand Up @@ -271,7 +271,7 @@ jobs:
curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}
curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/client-call

# Validation for app signals telemetry data
# Validation for application signals telemetry data
- name: Call endpoint and validate generated EMF logs
id: log-validation
if: steps.deploy-python-app.outcome == 'success' && !cancelled()
Expand Down Expand Up @@ -343,7 +343,7 @@ jobs:

# Clean up Procedures

- name: Clean Up App Signals
- name: Clean Up Application Signals
if: always()
continue-on-error: true
working-directory: enablement-script
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/appsignals-python-e2e-ec2-canary-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
## SPDX-License-Identifier: Apache-2.0

## This workflow aims to run the Application Signals Python end-to-end tests as a canary to
## test the artifacts for App Signals enablement. It will deploy a sample app and remote
## test the artifacts for Application Signals enablement. It will deploy a sample app and remote
## service on two EC2 instances, call the APIs, and validate the generated telemetry,
## including logs, metrics, and traces.
name: App Signals Enablement - Python E2E EC2 Canary Testing
Expand All @@ -24,7 +24,7 @@ jobs:
'ap-southeast-2','ap-southeast-3','ap-southeast-4','ca-central-1','eu-central-1','eu-central-2','eu-north-1',
'eu-south-1','eu-south-2','eu-west-1','eu-west-2','eu-west-3','il-central-1','me-central-1','me-south-1', 'sa-east-1',
'us-east-1','us-east-2', 'us-west-1', 'us-west-2']
uses: ./.github/workflows/appsignals-python-e2e-ec2-test.yml
uses: ./.github/workflows/application-signals-python-e2e-ec2-test.yml
secrets: inherit
with:
aws-region: ${{ matrix.aws-region }}
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/appsignals-python-e2e-eks-canary-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
## SPDX-License-Identifier: Apache-2.0

## This workflow aims to run the Application Signals Python end-to-end tests as a canary to
## test the artifacts for App Signals enablement. It will deploy a sample app and remote
## test the artifacts for Application Signals enablement. It will deploy a sample app and remote
## service onto an EKS cluster, call the APIs, and validate the generated telemetry,
## including logs, metrics, and traces.
name: App Signals Enablement - Python E2E EKS Canary Testing
Expand All @@ -29,9 +29,9 @@ jobs:
'ap-southeast-2','ap-southeast-3','ap-southeast-4','ca-central-1','eu-central-1','eu-central-2','eu-north-1',
'eu-south-1','eu-south-2','eu-west-1','eu-west-2','eu-west-3','il-central-1','me-central-1','me-south-1', 'sa-east-1',
'us-east-1','us-east-2', 'us-west-1', 'us-west-2']
uses: ./.github/workflows/appsignals-python-e2e-eks-test.yml
uses: ./.github/workflows/application-signals-python-e2e-eks-test.yml
secrets: inherit
with:
aws-region: ${{ matrix.aws-region }}
test-cluster-name: 'e2e-python-canary-test'
caller-workflow-name: 'appsignals-python-e2e-eks-canary-test'
caller-workflow-name: 'appsignals-python-e2e-eks-canary-test'
16 changes: 8 additions & 8 deletions .github/workflows/main_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,22 +86,22 @@ jobs:
pip install pytest
pytest contract-tests/tests

# AppSignals specific e2e eks tests
appsignals-python-e2e-eks-test:
# Application Signals specific e2e eks tests
application-signals-python-e2e-eks-test:
needs: [build]
uses: ./.github/workflows/appsignals-python-e2e-eks-test.yml
uses: ./.github/workflows/application-signals-python-e2e-eks-test.yml
secrets: inherit
with:
aws-region: ${{ needs.build.outputs.aws_default_region }}
test-cluster-name: e2e-python-adot-test
caller-workflow-name: 'main-build'
appsignals-adot-image: ${{ needs.build.outputs.staging_registry }}/aws-observability/adot-autoinstrumentation-python-staging
appsignals-adot-image-tag: ${{ needs.build.outputs.python_image_tag }}
application-signals-adot-image: ${{ needs.build.outputs.staging_registry }}/aws-observability/adot-autoinstrumentation-python-staging
application-signals-adot-image-tag: ${{ needs.build.outputs.python_image_tag }}

# AppSignals specific e2e tests for ec2
appsignals-python-e2e-ec2-test:
# Application Signals specific e2e tests for ec2
application-signals-python-e2e-ec2-test:
needs: [ build ]
uses: ./.github/workflows/appsignals-python-e2e-ec2-test.yml
uses: ./.github/workflows/application-signals-python-e2e-ec2-test.yml
secrets: inherit
with:
aws-region: ${{ needs.build.outputs.aws_default_region }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,11 @@
from opentelemetry.semconv.resource import ResourceAttributes
from opentelemetry.trace import set_tracer_provider

OTEL_AWS_APP_SIGNALS_ENABLED = "OTEL_AWS_APP_SIGNALS_ENABLED"
OTEL_METRIC_EXPORT_INTERVAL = "OTEL_METRIC_EXPORT_INTERVAL"
OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT = "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT"
APP_SIGNALS_ENABLED_CONFIG = "OTEL_AWS_APP_SIGNALS_ENABLED"
APPLICATION_SIGNALS_ENABLED_CONFIG = "OTEL_AWS_APPLICATION_SIGNALS_ENABLED"
APP_SIGNALS_EXPORTER_ENDPOINT_CONFIG = "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT"
APPLICATION_SIGNALS_EXPORTER_ENDPOINT_CONFIG = "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT"
METRIC_EXPORT_INTERVAL_CONFIG = "OTEL_METRIC_EXPORT_INTERVAL"
DEFAULT_METRIC_EXPORT_INTERVAL = 60000.0

_logger: Logger = getLogger(__name__)
Expand All @@ -74,8 +76,8 @@ class AwsOpenTelemetryConfigurator(_OTelSDKConfigurator):
- Add AttributePropagatingSpanProcessor to propagate span attributes from parent to child spans.
- Add AwsMetricAttributesSpanExporter to add more attributes to all spans.

You can control when these customizations are applied using the environment variable OTEL_AWS_APP_SIGNALS_ENABLED.
This flag is disabled by default.
You can control when these customizations are applied using the environment variable
OTEL_AWS_APPLICATION_SIGNALS_ENABLED. This flag is disabled by default.
"""

# pylint: disable=no-self-use
Expand Down Expand Up @@ -210,38 +212,38 @@ def _custom_import_sampler(sampler_name: str, resource: Resource) -> Sampler:


def _customize_sampler(sampler: Sampler) -> Sampler:
if not _is_app_signals_enabled():
if not _is_application_signals_enabled():
return sampler
return AlwaysRecordSampler(sampler)


def _customize_exporter(span_exporter: SpanExporter, resource: Resource) -> SpanExporter:
if not _is_app_signals_enabled():
if not _is_application_signals_enabled():
return span_exporter
return AwsMetricAttributesSpanExporterBuilder(span_exporter, resource).build()


def _customize_span_processors(provider: TracerProvider, resource: Resource) -> None:
if not _is_app_signals_enabled():
if not _is_application_signals_enabled():
return

# Construct and set local and remote attributes span processor
provider.add_span_processor(AttributePropagatingSpanProcessorBuilder().build())

# Construct meterProvider
_logger.info("AWS AppSignals enabled")
otel_metric_exporter = AppSignalsExporterProvider().create_exporter()
export_interval_millis = float(os.environ.get(OTEL_METRIC_EXPORT_INTERVAL, DEFAULT_METRIC_EXPORT_INTERVAL))
_logger.info("AWS Application Signals enabled")
otel_metric_exporter = ApplicationSignalsExporterProvider().create_exporter()
export_interval_millis = float(os.environ.get(METRIC_EXPORT_INTERVAL_CONFIG, DEFAULT_METRIC_EXPORT_INTERVAL))
_logger.debug("Span Metrics export interval: %s", export_interval_millis)
# Cap export interval to 60 seconds. This is currently required for metrics-trace correlation to work correctly.
if export_interval_millis > DEFAULT_METRIC_EXPORT_INTERVAL:
export_interval_millis = DEFAULT_METRIC_EXPORT_INTERVAL
_logger.info("AWS AppSignals metrics export interval capped to %s", export_interval_millis)
_logger.info("AWS Application Signals metrics export interval capped to %s", export_interval_millis)
periodic_exporting_metric_reader = PeriodicExportingMetricReader(
exporter=otel_metric_exporter, export_interval_millis=export_interval_millis
)
meter_provider: MeterProvider = MeterProvider(resource=resource, metric_readers=[periodic_exporting_metric_reader])
# Construct and set AppSignals metrics processor
# Construct and set application signals metrics processor
provider.add_span_processor(AwsSpanMetricsProcessorBuilder(meter_provider, resource).build())

return
Expand All @@ -254,12 +256,15 @@ def _customize_versions(auto_resource: Dict[str, any]) -> Dict[str, any]:
return auto_resource


def _is_app_signals_enabled():
return os.environ.get(OTEL_AWS_APP_SIGNALS_ENABLED, "false").lower() == "true"
def _is_application_signals_enabled():
return (
os.environ.get(APPLICATION_SIGNALS_ENABLED_CONFIG, os.environ.get(APP_SIGNALS_ENABLED_CONFIG, "false")).lower()
== "true"
)


class AppSignalsExporterProvider:
_instance: ClassVar["AppSignalsExporterProvider"] = None
class ApplicationSignalsExporterProvider:
_instance: ClassVar["ApplicationSignalsExporterProvider"] = None

def __new__(cls, *args, **kwargs):
if cls._instance is None:
Expand All @@ -271,11 +276,14 @@ def create_exporter(self):
protocol = os.environ.get(
OTEL_EXPORTER_OTLP_METRICS_PROTOCOL, os.environ.get(OTEL_EXPORTER_OTLP_PROTOCOL, "grpc")
)
_logger.debug("AppSignals export protocol: %s", protocol)
_logger.debug("AWS Application Signals export protocol: %s", protocol)

app_signals_endpoint = os.environ.get(OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT, "http://localhost:4315")
application_signals_endpoint = os.environ.get(
APPLICATION_SIGNALS_EXPORTER_ENDPOINT_CONFIG,
os.environ.get(APP_SIGNALS_EXPORTER_ENDPOINT_CONFIG, "http://localhost:4315"),
)

_logger.debug("AppSignals export endpoint: %s", app_signals_endpoint)
_logger.debug("AWS Application Signals export endpoint: %s", application_signals_endpoint)

temporality_dict: Dict[type, AggregationTemporality] = {}
for typ in [
Expand All @@ -290,8 +298,12 @@ def create_exporter(self):
temporality_dict[typ] = AggregationTemporality.DELTA

if protocol == "http/protobuf":
return OTLPHttpOTLPMetricExporter(endpoint=app_signals_endpoint, preferred_temporality=temporality_dict)
return OTLPHttpOTLPMetricExporter(
endpoint=application_signals_endpoint, preferred_temporality=temporality_dict
)
if protocol == "grpc":
return OTLPGrpcOTLPMetricExporter(endpoint=app_signals_endpoint, preferred_temporality=temporality_dict)
return OTLPGrpcOTLPMetricExporter(
endpoint=application_signals_endpoint, preferred_temporality=temporality_dict
)

raise RuntimeError(f"Unsupported AppSignals export protocol: {protocol} ")
raise RuntimeError(f"Unsupported AWS Application Signals export protocol: {protocol} ")
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
_customize_exporter,
_customize_sampler,
_customize_span_processors,
_is_app_signals_enabled,
_is_application_signals_enabled,
)
from amazon.opentelemetry.distro.aws_opentelemetry_distro import AwsOpenTelemetryDistro
from amazon.opentelemetry.distro.aws_span_metrics_processor import AwsSpanMetricsProcessor
Expand Down Expand Up @@ -202,50 +202,50 @@ def test_not_using_xray_sampler_does_not_modify_url_exclusion_env_vars(self):
self.assertEqual(os.environ.get("OTEL_PYTHON_REQUESTS_EXCLUDED_URLS", None), ",,,target_A,target_B,,,")
self.assertEqual(os.environ.get("OTEL_PYTHON_URLLIB3_EXCLUDED_URLS", None), "target_C,target_D")

def test_is_app_signals_enabled(self):
os.environ.setdefault("OTEL_AWS_APP_SIGNALS_ENABLED", "True")
self.assertTrue(_is_app_signals_enabled())
os.environ.pop("OTEL_AWS_APP_SIGNALS_ENABLED", None)
def test_is_application_signals_enabled(self):
os.environ.setdefault("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "True")
self.assertTrue(_is_application_signals_enabled())
os.environ.pop("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", None)

os.environ.setdefault("OTEL_AWS_APP_SIGNALS_ENABLED", "False")
self.assertFalse(_is_app_signals_enabled())
os.environ.pop("OTEL_AWS_APP_SIGNALS_ENABLED", None)
self.assertFalse(_is_app_signals_enabled())
os.environ.setdefault("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "False")
self.assertFalse(_is_application_signals_enabled())
os.environ.pop("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", None)
self.assertFalse(_is_application_signals_enabled())

def test_customize_sampler(self):
mock_sampler: Sampler = MagicMock()
customized_sampler: Sampler = _customize_sampler(mock_sampler)
self.assertEqual(mock_sampler, customized_sampler)

os.environ.setdefault("OTEL_AWS_APP_SIGNALS_ENABLED", "True")
os.environ.setdefault("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "True")
customized_sampler = _customize_sampler(mock_sampler)
self.assertNotEqual(mock_sampler, customized_sampler)
self.assertIsInstance(customized_sampler, AlwaysRecordSampler)
self.assertEqual(mock_sampler, customized_sampler._root_sampler)
os.environ.pop("OTEL_AWS_APP_SIGNALS_ENABLED", None)
os.environ.pop("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", None)

def test_customize_exporter(self):
mock_exporter: SpanExporter = MagicMock()
customized_exporter: SpanExporter = _customize_exporter(mock_exporter, Resource.get_empty())
self.assertEqual(mock_exporter, customized_exporter)

os.environ.setdefault("OTEL_AWS_APP_SIGNALS_ENABLED", "True")
os.environ.setdefault("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "True")
customized_exporter = _customize_exporter(mock_exporter, Resource.get_empty())
self.assertNotEqual(mock_exporter, customized_exporter)
self.assertIsInstance(customized_exporter, AwsMetricAttributesSpanExporter)
self.assertEqual(mock_exporter, customized_exporter._delegate)
os.environ.pop("OTEL_AWS_APP_SIGNALS_ENABLED", None)
os.environ.pop("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", None)

def test_customize_span_processors(self):
mock_tracer_provider: TracerProvider = MagicMock()
_customize_span_processors(mock_tracer_provider, Resource.get_empty())
self.assertEqual(mock_tracer_provider.add_span_processor.call_count, 0)

os.environ.setdefault("OTEL_AWS_APP_SIGNALS_ENABLED", "True")
os.environ.setdefault("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "True")
_customize_span_processors(mock_tracer_provider, Resource.get_empty())
self.assertEqual(mock_tracer_provider.add_span_processor.call_count, 2)
first_processor: SpanProcessor = mock_tracer_provider.add_span_processor.call_args_list[0].args[0]
self.assertIsInstance(first_processor, AttributePropagatingSpanProcessor)
second_processor: SpanProcessor = mock_tracer_provider.add_span_processor.call_args_list[1].args[0]
self.assertIsInstance(second_processor, AwsSpanMetricsProcessor)
os.environ.pop("OTEL_AWS_APP_SIGNALS_ENABLED", None)
os.environ.pop("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", None)
Loading