Skip to content

Commit beb2b5d

Browse files
harrryrharrryr
committed
Remove Public Endpoint from K8s Platform
1 parent 8cdc51d commit beb2b5d

32 files changed

+171
-381
lines changed

‎.github/workflows/java-k8s-e2e-test.yml

Lines changed: 5 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -125,35 +125,11 @@ jobs:
125125
-var="patch_image_arn=${{ env.PATCH_IMAGE_ARN }}" \
126126
-var="release_testing_ecr_account=${{ env.RELEASE_TESTING_ECR_ACCOUNT }}"
127127
128-
- name: Get Remote Service IP
128+
- name: Get Main and Remote Service IP
129129
run: |
130+
echo MAIN_SERVICE_IP="$(aws ssm get-parameter --region ${{ env.E2E_TEST_AWS_REGION }} --name main-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
130131
echo REMOTE_SERVICE_IP="$(aws ssm get-parameter --region ${{ env.E2E_TEST_AWS_REGION }} --name remote-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
131132
132-
- name: Wait for app endpoint to come online
133-
id: endpoint-check
134-
run: |
135-
attempt_counter=0
136-
max_attempts=30
137-
until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/); do
138-
if [ ${attempt_counter} -eq ${max_attempts} ];then
139-
echo "Max attempts reached"
140-
exit 1
141-
fi
142-
143-
printf '.'
144-
attempt_counter=$(($attempt_counter+1))
145-
sleep 10
146-
done
147-
# This steps increases the speed of the validation by creating the telemetry data in advance
148-
# It is run after the gradle build to give the app time to initialize after the pods become ready
149-
- name: Call all test APIs
150-
continue-on-error: true
151-
run: |
152-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/outgoing-http-call"; echo
153-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
154-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
155-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/client-call"; echo
156-
157133
- name: Initiate Gradlew Daemon
158134
if: steps.initiate-gradlew == 'failure'
159135
uses: ./.github/workflows/actions/execute_and_retry
@@ -169,7 +145,7 @@ jobs:
169145
id: log-validation
170146
run: ./gradlew validator:run --args='-c java/k8s/log-validation.yml
171147
--testing-id ${{ env.TESTING_ID }}
172-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
148+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8080
173149
--region ${{ env.E2E_TEST_AWS_REGION }}
174150
--account-id ${{ env.ACCOUNT_ID }}
175151
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -186,7 +162,7 @@ jobs:
186162
if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
187163
run: ./gradlew validator:run --args='-c java/k8s/metric-validation.yml
188164
--testing-id ${{ env.TESTING_ID }}
189-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
165+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8080
190166
--region ${{ env.E2E_TEST_AWS_REGION }}
191167
--account-id ${{ env.ACCOUNT_ID }}
192168
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -204,7 +180,7 @@ jobs:
204180
if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
205181
run: ./gradlew validator:run --args='-c java/k8s/trace-validation.yml
206182
--testing-id ${{ env.TESTING_ID }}
207-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
183+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8080
208184
--region ${{ env.E2E_TEST_AWS_REGION }}
209185
--account-id ${{ env.ACCOUNT_ID }}
210186
--metric-namespace ${{ env.METRIC_NAMESPACE }}

‎.github/workflows/python-k8s-e2e-test.yml

Lines changed: 5 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -125,36 +125,11 @@ jobs:
125125
-var="patch_image_arn=${{ env.PATCH_IMAGE_ARN }}" \
126126
-var="release_testing_ecr_account=${{ env.RELEASE_TESTING_ECR_ACCOUNT }}"
127127
128-
- name: Get Remote Service IP
128+
- name: Get Main and Remote Service IP
129129
run: |
130+
echo MAIN_SERVICE_IP="$(aws ssm get-parameter --region ${{ env.E2E_TEST_AWS_REGION }} --name python-main-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
130131
echo REMOTE_SERVICE_IP="$(aws ssm get-parameter --region us-east-1 --name python-remote-service-ip-${{ env.TESTING_ID }} | jq -r '.Parameter.Value')" >> $GITHUB_ENV
131132
132-
- name: Wait for app endpoint to come online
133-
id: endpoint-check
134-
run: |
135-
attempt_counter=0
136-
max_attempts=30
137-
until $(curl --output /dev/null --silent --head --fail http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/); do
138-
if [ ${attempt_counter} -eq ${max_attempts} ];then
139-
echo "Max attempts reached"
140-
exit 1
141-
fi
142-
143-
printf '.'
144-
attempt_counter=$(($attempt_counter+1))
145-
sleep 10
146-
done
147-
148-
# This steps increases the speed of the validation by creating the telemetry data in advance
149-
# It is run after the gradle build to give the app time to initialize after the pods become ready
150-
- name: Call all test APIs
151-
continue-on-error: true
152-
run: |
153-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/outgoing-http-call"; echo
154-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
155-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}"; echo
156-
curl -S -s "http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100/client-call"; echo
157-
158133
- name: Initiate Gradlew Daemon
159134
if: steps.initiate-gradlew == 'failure'
160135
uses: ./.github/workflows/actions/execute_and_retry
@@ -170,7 +145,7 @@ jobs:
170145
id: log-validation
171146
run: ./gradlew validator:run --args='-c python/k8s/log-validation.yml
172147
--testing-id ${{ env.TESTING_ID }}
173-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
148+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8000
174149
--region ${{ env.E2E_TEST_AWS_REGION }}
175150
--account-id ${{ env.ACCOUNT_ID }}
176151
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -187,7 +162,7 @@ jobs:
187162
if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
188163
run: ./gradlew validator:run --args='-c python/k8s/metric-validation.yml
189164
--testing-id ${{ env.TESTING_ID }}
190-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
165+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8000
191166
--region ${{ env.E2E_TEST_AWS_REGION }}
192167
--account-id ${{ env.ACCOUNT_ID }}
193168
--metric-namespace ${{ env.METRIC_NAMESPACE }}
@@ -205,7 +180,7 @@ jobs:
205180
if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
206181
run: ./gradlew validator:run --args='-c python/k8s/trace-validation.yml
207182
--testing-id ${{ env.TESTING_ID }}
208-
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}:30100
183+
--endpoint http://${{ env.MAIN_SERVICE_IP }}:8000
209184
--region ${{ env.E2E_TEST_AWS_REGION }}
210185
--account-id ${{ env.ACCOUNT_ID }}
211186
--metric-namespace ${{ env.METRIC_NAMESPACE }}

‎terraform/java/k8s/cleanup/main.tf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ resource "null_resource" "cleanup" {
3232
echo "LOG: Printing cluster state after cleanup"
3333
kubectl get pods -A
3434
35-
# Delete ssm parameter for remote service ip
35+
# Delete ssm parameter for main and remote service ip
36+
aws ssm delete-parameter --name main-service-ip-${var.test_id}
3637
aws ssm delete-parameter --name remote-service-ip-${var.test_id}
3738
EOF
3839
]

‎terraform/java/k8s/deploy/main.tf

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,19 +135,35 @@ resource "null_resource" "deploy" {
135135
kubectl apply -f frontend-service-depl.yaml
136136
kubectl apply -f remote-service-depl.yaml
137137
138-
# Expose sample app on port 30100
139-
echo "LOG: Exposing main sample app on port 30100"
140-
kubectl expose deployment sample-app-deployment-${var.test_id} -n sample-app-namespace --type="NodePort" --port 8080
141-
kubectl patch service sample-app-deployment-${var.test_id} -n sample-app-namespace --type='json' --patch='[{"op": "replace", "path": "/spec/ports/0/nodePort", "value":30100}]'
142-
143138
# Wait for sample app to be reach ready state
144139
sleep 10
145140
kubectl wait --for=condition=Ready --request-timeout '5m' pod --all -n sample-app-namespace
146141
147-
# Emit remote service pod IP
142+
# Emit main and remote service pod IP
148143
echo "LOG: Outputting remote service pod IP to SSM using put-parameter API"
144+
aws ssm put-parameter --region ${var.aws_region} --name main-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pods -n sample-app-namespace --selector=app=sample-app -o jsonpath='{.items[0].status.podIP}')
149145
aws ssm put-parameter --region ${var.aws_region} --name remote-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pod --selector=app=remote-app -n sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
150146
147+
# Deploy the traffic generator
148+
kubectl create deployment -n sample-app-namespace traffic-generator \
149+
--image=$ACCOUNT.dkr.ecr.${var.aws_region}.amazonaws.com/e2e-test-resource:traffic-generator \
150+
--replicas=1
151+
152+
# Patch it with ImagePull always policy so that it pulls the latest image from the ECR
153+
kubectl patch deployment -n sample-app-namespace traffic-generator --patch '{"spec": {"template": {"spec": {"containers": [{"name": "e2e-test-resource", "imagePullPolicy": "Always"}]}}}}'
154+
kubectl patch deployment traffic-generator -n sample-app-namespace --type='json' -p='[{"op": "add", "path": "/spec/template/spec/imagePullSecrets", "value": [{"name": "ecr-secret"}]}]'
155+
156+
# Add the appropriate environment variables to the traffic generator
157+
kubectl set env -n sample-app-namespace deployment/traffic-generator MAIN_ENDPOINT=$(kubectl get pods -n sample-app-namespace --selector=app=sample-app -o jsonpath='{.items[0].status.podIP}'):8080
158+
kubectl set env -n sample-app-namespace deployment/traffic-generator REMOTE_ENDPOINT=$(kubectl get pod --selector=app=remote-app -n sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
159+
kubectl set env -n sample-app-namespace deployment/traffic-generator ID=${var.test_id}
160+
kubectl set env -n sample-app-namespace deployment/traffic-generator CANARY_TYPE=java-k8s
161+
162+
# Restart the traffic generator with the new configuration
163+
kubectl get pods -n sample-app-namespace --no-headers | grep '^traffic-generator' | awk '{print $1}' | xargs kubectl delete pod -n sample-app-namespace
164+
165+
sleep 10
166+
151167
EOF
152168
]
153169
}

‎terraform/python/k8s/cleanup/main.tf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ resource "null_resource" "cleanup" {
4646
echo "LOG: Printing cluster state after cleanup"
4747
kubectl get pods -A
4848
49-
# Delete ssm parameter for remote service ip
49+
# Delete ssm parameter for main and remote service ip
50+
aws ssm delete-parameter --name python-main-service-ip-${var.test_id}
5051
aws ssm delete-parameter --name python-remote-service-ip-${var.test_id}
5152
5253
EOF

‎terraform/python/k8s/deploy/main.tf

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,18 +135,34 @@ resource "null_resource" "deploy" {
135135
kubectl apply -f python-frontend-service-depl.yaml
136136
kubectl apply -f python-remote-service-depl.yaml
137137
138-
# Expose sample app on port 30100
139-
echo "LOG: Exposing main sample app on port 30100"
140-
kubectl expose deployment python-sample-app-deployment-${var.test_id} -n python-sample-app-namespace --type="NodePort" --port 8000
141-
kubectl patch service python-sample-app-deployment-${var.test_id} -n python-sample-app-namespace --type='json' --patch='[{"op": "replace", "path": "/spec/ports/0/nodePort", "value":30100}]'
142-
143138
echo "Wait for sample app to be reach ready state"
144139
sleep 10
145140
kubectl wait --for=condition=Ready --request-timeout '10m' pod --all -n python-sample-app-namespace
146141
147-
# Emit remote service pod IP
142+
# Emit main and remote service pod IP
148143
echo "LOG: Outputting remote service pod IP to SSM using put-parameter API"
144+
aws ssm put-parameter --region ${var.aws_region} --name python-main-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pod --selector=app=python-sample-app -n python-sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
149145
aws ssm put-parameter --region ${var.aws_region} --name python-remote-service-ip-${var.test_id} --type String --overwrite --value $(kubectl get pod --selector=app=python-remote-app -n python-sample-app-namespace -o jsonpath='{.items[0].status.podIP}')
146+
147+
# Deploy the traffic generator
148+
kubectl create deployment -n python-sample-app-namespace traffic-generator \
149+
--image=$ACCOUNT.dkr.ecr.${var.aws_region}.amazonaws.com/e2e-test-resource:traffic-generator \
150+
--replicas=1
151+
152+
# Patch it with ImagePull always policy so that it pulls the latest image from the ECR
153+
kubectl patch deployment -n python-sample-app-namespace traffic-generator --patch '{"spec": {"template": {"spec": {"containers": [{"name": "e2e-test-resource", "imagePullPolicy": "Always"}]}}}}'
154+
kubectl patch deployment traffic-generator -n python-sample-app-namespace --type='json' -p='[{"op": "add", "path": "/spec/template/spec/imagePullSecrets", "value": [{"name": "ecr-secret"}]}]'
155+
156+
# Add the appropriate environment variables to the traffic generator
157+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator MAIN_ENDPOINT=$(kubectl get pods -n python-sample-app-namespace --selector=app=python-sample-app -o jsonpath='{.items[0].status.podIP}'):8000
158+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator REMOTE_ENDPOINT=$(kubectl get pod -n python-sample-app-namespace --selector=app=python-remote-app -o jsonpath='{.items[0].status.podIP}')
159+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator ID=${var.test_id}
160+
kubectl set env -n python-sample-app-namespace deployment/traffic-generator CANARY_TYPE=python-k8s
161+
162+
# Restart the traffic generator with the new configuration
163+
kubectl get pods -n python-sample-app-namespace --no-headers | grep '^traffic-generator' | awk '{print $1}' | xargs kubectl delete pod -n python-sample-app-namespace
164+
165+
sleep 10
150166
EOF
151167
]
152168
}

‎validator/src/main/java/com/amazon/aoc/callers/HttpCaller.java

Lines changed: 0 additions & 102 deletions
This file was deleted.

‎validator/src/main/java/com/amazon/aoc/callers/ICaller.java

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎validator/src/main/java/com/amazon/aoc/helpers/CWMetricHelper.java

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
package com.amazon.aoc.helpers;
1717

18-
import com.amazon.aoc.callers.ICaller;
1918
import com.amazon.aoc.fileconfigs.FileConfig;
2019
import com.amazon.aoc.models.Context;
2120
import com.amazonaws.services.cloudwatch.model.Dimension;
@@ -38,17 +37,11 @@ public class CWMetricHelper {
3837
*
3938
* @param context testing context
4039
* @param expectedMetric expected template
41-
* @param caller http caller, none caller, could be null
4240
* @return list of metrics
4341
* @throws Exception when caller throws exception or template can not be found
4442
*/
4543
public List<Metric> listExpectedMetrics(
46-
Context context, FileConfig expectedMetric, ICaller caller) throws Exception {
47-
// call endpoint
48-
if (caller != null) {
49-
caller.callSampleApp();
50-
}
51-
44+
Context context, FileConfig expectedMetric) throws Exception {
5245
// get expected metrics as yaml from config
5346
String yamlExpectedMetrics = mustacheHelper.render(expectedMetric, context);
5447

0 commit comments

Comments
 (0)