Skip to content

Commit 506fc2d

Browse files
harrryrharrryr
committed
Remove Public Endpoint from K8s Platform
1 parent 8cdc51d commit 506fc2d

File tree

27 files changed

+188
-355
lines changed

27 files changed

+188
-355
lines changed

‎.github/workflows/java-k8s-e2e-test.yml

Lines changed: 12 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ on:
2121
required: false
2222
type: string
2323

24-
concurrency:
25-
group: '${{ github.workflow }} @ ${{ inputs.aws-region }}'
26-
cancel-in-progress: false
24+
#concurrency:
25+
# group: '${{ github.workflow }} @ ${{ inputs.aws-region }}'
26+
# cancel-in-progress: false
2727

2828
permissions:
2929
id-token: write
@@ -40,6 +40,8 @@ env:
4040
METRIC_NAMESPACE: ApplicationSignals
4141
LOG_GROUP_NAME: /aws/application-signals/data
4242
TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE}
43+
MAIN_SERVICE_ENDPOINT: ${{ secrets.TEMP_IAD_ENDPOINT_K8S }}
44+
MASTER_NODE_SSH_KEY: ${{ secrets.TEMP_IAD_SSH_KEY_K8S }}
4345

4446
jobs:
4547
java-k8s:
@@ -82,8 +84,8 @@ jobs:
8284
JAVA_MAIN_SAMPLE_APP_IMAGE, e2e-test/java-main-sample-app-image
8385
JAVA_REMOTE_SAMPLE_APP_IMAGE, e2e-test/java-remote-sample-app-image
8486
RELEASE_TESTING_ECR_ACCOUNT, e2e-test/${{ github.event.repository.name }}/java-k8s-release-testing-account
85-
MAIN_SERVICE_ENDPOINT, e2e-test/${{ github.event.repository.name }}/java-k8s-master-node-endpoint
86-
MASTER_NODE_SSH_KEY, e2e-test/${{ github.event.repository.name }}/java-k8s-ssh-key
87+
# MAIN_SERVICE_ENDPOINT, e2e-test/${{ github.event.repository.name }}/java-k8s-master-node-endpoint
88+
# MASTER_NODE_SSH_KEY, e2e-test/${{ github.event.repository.name }}/java-k8s-ssh-key
8789

8890
- name: Prepare and upload sample app deployment files
8991
working-directory: terraform/java/k8s/deploy/resources
@@ -125,35 +127,11 @@ jobs:
125127
-var="patch_image_arn=${{ env.PATCH_IMAGE_ARN }}" \
126128
-var="release_testing_ecr_account=${{ env.RELEASE_TESTING_ECR_ACCOUNT }}"
127129
128-
- name: Get Remote Service IP
130+
- name: Get Main and Remote Service IP
129131
run: |
132+
echo MAIN_SERVICE_IP="$(aws ssm get-parameter --region ${{ env.E2E_TEST_AWS_REGION }} --name main-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
130133
echo REMOTE_SERVICE_IP="$(aws ssm get-parameter --region ${{ env.E2E_TEST_AWS_REGION }} --name remote-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
131134
132-
- name: Wait for app endpoint to come online
133-
id: endpoint-check
134-
run: |
135-
attempt_counter=0
136-
max_attempts=30
137-
until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/); do
138-
if [ ${attempt_counter} -eq ${max_attempts} ];then
139-
echo "Max attempts reached"
140-
exit 1
141-
fi
142-
143-
printf '.'
144-
attempt_counter=$(($attempt_counter+1))
145-
sleep 10
146-
done
147-
# This steps increases the speed of the validation by creating the telemetry data in advance
148-
# It is run after the gradle build to give the app time to initialize after the pods become ready
149-
- name: Call all test APIs
150-
continue-on-error: true
151-
run: |
152-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/outgoing-http-call"; echo
153-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
154-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
155-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/client-call"; echo
156-
157135
- name: Initiate Gradlew Daemon
158136
if: steps.initiate-gradlew == 'failure'
159137
uses: ./.github/workflows/actions/execute_and_retry
@@ -169,7 +147,7 @@ jobs:
169147
id: log-validation
170148
run: ./gradlew validator:run --args='-c java/k8s/log-validation.yml
171149
--testing-id ${{ env.TESTING_ID }}
172-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
150+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8080
173151
--region ${{ env.E2E_TEST_AWS_REGION }}
174152
--account-id ${{ env.ACCOUNT_ID }}
175153
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -186,7 +164,7 @@ jobs:
186164
if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
187165
run: ./gradlew validator:run --args='-c java/k8s/metric-validation.yml
188166
--testing-id ${{ env.TESTING_ID }}
189-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
167+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8080
190168
--region ${{ env.E2E_TEST_AWS_REGION }}
191169
--account-id ${{ env.ACCOUNT_ID }}
192170
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -204,7 +182,7 @@ jobs:
204182
if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
205183
run: ./gradlew validator:run --args='-c java/k8s/trace-validation.yml
206184
--testing-id ${{ env.TESTING_ID }}
207-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
185+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8080
208186
--region ${{ env.E2E_TEST_AWS_REGION }}
209187
--account-id ${{ env.ACCOUNT_ID }}
210188
--metric-namespace ${{ env.METRIC_NAMESPACE }}

‎.github/workflows/python-k8s-e2e-test.yml

Lines changed: 9 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ env:
4040
METRIC_NAMESPACE: ApplicationSignals
4141
LOG_GROUP_NAME: /aws/application-signals/data
4242
TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE}
43+
MAIN_SERVICE_ENDPOINT: ${{ secrets.TEMP_IAD_ENDPOINT_K8S }}
44+
MASTER_NODE_SSH_KEY: ${{ secrets.TEMP_IAD_SSH_KEY_K8S }}
4345

4446
jobs:
4547
python-k8s:
@@ -82,8 +84,8 @@ jobs:
8284
PYTHON_MAIN_SAMPLE_APP_IMAGE, e2e-test/python-main-sample-app-image
8385
PYTHON_REMOTE_SAMPLE_APP_IMAGE, e2e-test/python-remote-sample-app-image
8486
RELEASE_TESTING_ECR_ACCOUNT, e2e-test/${{ github.event.repository.name }}/python-k8s-release-testing-account
85-
MAIN_SERVICE_ENDPOINT, e2e-test/${{ github.event.repository.name }}/python-k8s-master-node-endpoint
86-
MASTER_NODE_SSH_KEY, e2e-test/${{ github.event.repository.name }}/python-k8s-ssh-key
87+
# MAIN_SERVICE_ENDPOINT, e2e-test/${{ github.event.repository.name }}/python-k8s-master-node-endpoint
88+
# MASTER_NODE_SSH_KEY, e2e-test/${{ github.event.repository.name }}/python-k8s-ssh-key
8789

8890
- name: Prepare and upload sample app deployment files
8991
working-directory: terraform/python/k8s/deploy/resources
@@ -125,36 +127,11 @@ jobs:
125127
-var="patch_image_arn=${{ env.PATCH_IMAGE_ARN }}" \
126128
-var="release_testing_ecr_account=${{ env.RELEASE_TESTING_ECR_ACCOUNT }}"
127129
128-
- name: Get Remote Service IP
130+
- name: Get Main and Remote Service IP
129131
run: |
132+
echo MAIN_SERVICE_IP="$(aws ssm get-parameter --region ${{ env.E2E_TEST_AWS_REGION }} --name python-main-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
130133
echo REMOTE_SERVICE_IP="$(aws ssm get-parameter --region us-east-1 --name python-remote-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
131134
132-
- name: Wait for app endpoint to come online
133-
id: endpoint-check
134-
run: |
135-
attempt_counter=0
136-
max_attempts=30
137-
until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/); do
138-
if [ ${attempt_counter} -eq ${max_attempts} ];then
139-
echo "Max attempts reached"
140-
exit 1
141-
fi
142-
143-
printf '.'
144-
attempt_counter=$(($attempt_counter+1))
145-
sleep 10
146-
done
147-
148-
# This steps increases the speed of the validation by creating the telemetry data in advance
149-
# It is run after the gradle build to give the app time to initialize after the pods become ready
150-
- name: Call all test APIs
151-
continue-on-error: true
152-
run: |
153-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/outgoing-http-call"; echo
154-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
155-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
156-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/client-call"; echo
157-
158135
- name: Initiate Gradlew Daemon
159136
if: steps.initiate-gradlew == 'failure'
160137
uses: ./.github/workflows/actions/execute_and_retry
@@ -170,7 +147,7 @@ jobs:
170147
id: log-validation
171148
run: ./gradlew validator:run --args='-c python/k8s/log-validation.yml
172149
--testing-id ${{ env.TESTING_ID }}
173-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
150+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8000
174151
--region ${{ env.E2E_TEST_AWS_REGION }}
175152
--account-id ${{ env.ACCOUNT_ID }}
176153
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -187,7 +164,7 @@ jobs:
187164
if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
188165
run: ./gradlew validator:run --args='-c python/k8s/metric-validation.yml
189166
--testing-id ${{ env.TESTING_ID }}
190-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
167+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8000
191168
--region ${{ env.E2E_TEST_AWS_REGION }}
192169
--account-id ${{ env.ACCOUNT_ID }}
193170
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -205,7 +182,7 @@ jobs:
205182
if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
206183
run: ./gradlew validator:run --args='-c python/k8s/trace-validation.yml
207184
--testing-id ${{ env.TESTING_ID }}
208-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
185+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8000
209186
--region ${{ env.E2E_TEST_AWS_REGION }}
210187
--account-id ${{ env.ACCOUNT_ID }}
211188
--metric-namespace ${{ env.METRIC_NAMESPACE }}

‎.github/workflows/test.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
## SPDX-License-Identifier: Apache-2.0
3+
4+
## This workflow aims to run the Application Signals end-to-end tests as a canary to
5+
## test the artifacts for App Signals enablement. It will deploy a sample app and remote
6+
## service on two EC2 instances, call the APIs, and validate the generated telemetry,
7+
## including logs, metrics, and traces.
8+
name: Test
9+
on:
10+
workflow_dispatch:
11+
push:
12+
13+
permissions:
14+
id-token: write
15+
contents: read
16+
17+
jobs:
18+
java-k8s:
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
aws-region: [ 'us-east-1' ]
23+
uses: ./.github/workflows/java-k8s-e2e-test.yml
24+
secrets: inherit
25+
with:
26+
aws-region: ${{ matrix.aws-region }}
27+
caller-workflow-name: 'test'
28+
29+
python-k8s:
30+
needs: java-k8s
31+
strategy:
32+
fail-fast: false
33+
matrix:
34+
aws-region: [ 'us-east-1' ]
35+
uses: ./.github/workflows/python-k8s-e2e-test.yml
36+
secrets: inherit
37+
with:
38+
aws-region: ${{ matrix.aws-region }}
39+
caller-workflow-name: 'test'
40+

‎terraform/java/k8s/cleanup/main.tf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ resource "null_resource" "cleanup" {
3232
echo "LOG: Printing cluster state after cleanup"
3333
kubectl get pods -A
3434
35-
# Delete ssm parameter for remote service ip
35+
# Delete ssm parameter for main and remote service ip
36+
aws ssm delete-parameter --name main-service-ip-${var.test_id}
3637
aws ssm delete-parameter --name remote-service-ip-${var.test_id}
3738
EOF
3839
]

‎terraform/java/k8s/deploy/main.tf

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,19 +135,35 @@ resource "null_resource" "deploy" {
135135
kubectl apply -f frontend-service-depl.yaml
136136
kubectl apply -f remote-service-depl.yaml
137137
138-
# Expose sample app on port 30100
139-
echo "LOG: Exposing main sample app on port 30100"
140-
kubectl expose deployment sample-app-deployment-${var.test_id} -n sample-app-namespace --type="NodePort" --port 8080
141-
kubectl patch service sample-app-deployment-${var.test_id} -n sample-app-namespace --type='json' --patch='[{"op": "replace", "path": "/spec/ports/0/nodePort", "value":30100}]'
142-
143138
# Wait for sample app to be reach ready state
144139
sleep 10
145140
kubectl wait --for=condition=Ready --request-timeout '5m' pod --all -n sample-app-namespace
146141
147-
# Emit remote service pod IP
142+
# Emit main and remote service pod IP
148143
echo "LOG: Outputting remote service pod IP to SSM using put-parameter API"
144+
aws ssm put-parameter --region ${var.aws_region} --name main-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pods -n sample-app-namespace --selector=app=sample-app -o jsonpath='{.items[0].status.podIP}')
149145
aws ssm put-parameter --region ${var.aws_region} --name remote-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pod --selector=app=remote-app -n sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
150146
147+
# Deploy the traffic generator
148+
kubectl create deployment -n sample-app-namespace traffic-generator \
149+
--image=$ACCOUNT.dkr.ecr.${var.aws_region}.amazonaws.com/e2e-test-resource:traffic-generator \
150+
--replicas=1
151+
152+
# Patch it with ImagePull always policy so that it pulls the latest image from the ECR
153+
kubectl patch deployment -n sample-app-namespace traffic-generator --patch '{"spec": {"template": {"spec": {"containers": [{"name": "e2e-test-resource", "imagePullPolicy": "Always"}]}}}}'
154+
kubectl patch deployment traffic-generator -n sample-app-namespace --type='json' -p='[{"op": "add", "path": "/spec/template/spec/imagePullSecrets", "value": [{"name": "ecr-secret"}]}]'
155+
156+
# Add the appropriate environment variables to the traffic generator
157+
kubectl set env -n sample-app-namespace deployment/traffic-generator MAIN_ENDPOINT=$(kubectl get pods -n sample-app-namespace --selector=app=sample-app -o jsonpath='{.items[0].status.podIP}'):8080
158+
kubectl set env -n sample-app-namespace deployment/traffic-generator REMOTE_ENDPOINT=$(kubectl get pod --selector=app=remote-app -n sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
159+
kubectl set env -n sample-app-namespace deployment/traffic-generator ID=${var.test_id}
160+
kubectl set env -n sample-app-namespace deployment/traffic-generator CANARY_TYPE=java-k8s
161+
162+
# Restart the traffic generator with the new configuration
163+
kubectl get pods -n sample-app-namespace --no-headers | grep '^traffic-generator' | awk '{print $1}' | xargs kubectl delete pod -n sample-app-namespace
164+
165+
sleep 10
166+
151167
EOF
152168
]
153169
}

‎terraform/python/k8s/cleanup/main.tf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ resource "null_resource" "cleanup" {
4646
echo "LOG: Printing cluster state after cleanup"
4747
kubectl get pods -A
4848
49-
# Delete ssm parameter for remote service ip
49+
# Delete ssm parameter for main and remote service ip
50+
aws ssm delete-parameter --name python-main-service-ip-${var.test_id}
5051
aws ssm delete-parameter --name python-remote-service-ip-${var.test_id}
5152
5253
EOF

‎terraform/python/k8s/deploy/main.tf

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,18 +135,34 @@ resource "null_resource" "deploy" {
135135
kubectl apply -f python-frontend-service-depl.yaml
136136
kubectl apply -f python-remote-service-depl.yaml
137137
138-
# Expose sample app on port 30100
139-
echo "LOG: Exposing main sample app on port 30100"
140-
kubectl expose deployment python-sample-app-deployment-${var.test_id} -n python-sample-app-namespace --type="NodePort" --port 8000
141-
kubectl patch service python-sample-app-deployment-${var.test_id} -n python-sample-app-namespace --type='json' --patch='[{"op": "replace", "path": "/spec/ports/0/nodePort", "value":30100}]'
142-
143138
echo "Wait for sample app to be reach ready state"
144139
sleep 10
145140
kubectl wait --for=condition=Ready --request-timeout '10m' pod --all -n python-sample-app-namespace
146141
147-
# Emit remote service pod IP
142+
# Emit main and remote service pod IP
148143
echo "LOG: Outputting remote service pod IP to SSM using put-parameter API"
144+
aws ssm put-parameter --region ${var.aws_region} --name python-main-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pod --selector=app=python-sample-app -n python-sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
149145
aws ssm put-parameter --region ${var.aws_region} --name python-remote-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pod --selector=app=python-remote-app -n python-sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
146+
147+
# Deploy the traffic generator
148+
kubectl create deployment -n python-sample-app-namespace traffic-generator \
149+
--image=$ACCOUNT.dkr.ecr.${var.aws_region}.amazonaws.com/e2e-test-resource:traffic-generator \
150+
--replicas=1
151+
152+
# Patch it with ImagePull always policy so that it pulls the latest image from the ECR
153+
kubectl patch deployment -n python-sample-app-namespace traffic-generator --patch '{"spec": {"template": {"spec": {"containers": [{"name": "e2e-test-resource", "imagePullPolicy": "Always"}]}}}}'
154+
kubectl patch deployment traffic-generator -n python-sample-app-namespace --type='json' -p='[{"op": "add", "path": "/spec/template/spec/imagePullSecrets", "value": [{"name": "ecr-secret"}]}]'
155+
156+
# Add the appropriate environment variables to the traffic generator
157+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator MAIN_ENDPOINT=$(kubectl get pods -n python-sample-app-namespace --selector=app=python-sample-app -o jsonpath='{.items[0].status.podIP}'):8000
158+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator REMOTE_ENDPOINT=$(kubectl get pod -n python-sample-app-namespace --selector=app=python-remote-app -o jsonpath='{.items[0].status.podIP}')
159+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator ID=${var.test_id}
160+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator CANARY_TYPE=python-k8s
161+
162+
# Restart the traffic generator with the new configuration
163+
kubectl get pods -n python-sample-app-namespace --no-headers | grep '^traffic-generator' | awk '{print $1}' | xargs kubectl delete pod -n python-sample-app-namespace
164+
165+
sleep 10
150166
EOF
151167
]
152168
}

0 commit comments

Comments
 (0)