Delete Test Resources for Device Advisor (#285)

* update DA script: now it uploads the logs to s3 before delete the testing thing.

* make sure delete test resource before exit

* add backoff for device advisor request

* update the print statement for logs path

Author: xiazhvera

builder.json

@@ -17,7 +17,8 @@
         "DA_TOPIC": "test/da",
         "DA_SHADOW_PROPERTY": "datest",
         "DA_SHADOW_VALUE_SET": "ON",
-        "DA_SHADOW_VALUE_DEFAULT": "OFF"
+        "DA_SHADOW_VALUE_DEFAULT": "OFF",
+        "DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs"
     },
     "hosts": {
         "ubuntu": {

deviceadvisor/script/DATestRun.py

@@ -3,7 +3,8 @@
 import json
 import os
 import subprocess
-import platform
+import re
+import random
 from time import sleep
 
 ##############################################
@@ -20,13 +21,37 @@ def delete_thing_with_certi(thingName, certiId, certiArn):
     os.remove(os.environ["DA_CERTI"])
     os.remove(os.environ["DA_KEY"])
 
+# Export the testing log and upload it to S3 bucket
+def process_logs(log_group, log_stream, thing_name):
+    logs_client = boto3.client('logs')
+    response = logs_client.get_log_events(
+        logGroupName=log_group,
+        logStreamName=log_stream
+    )
+    log_file = thing_name + ".log"
+    f = open(log_file, 'w')
+    for event in response["events"]:
+        f.write(event['message'])
+    f.close()
+    s3.Bucket(os.environ['DA_S3_NAME']).upload_file(log_file, log_file)
+    os.remove(log_file)
+    print("[Device Advisor] Device Advisor Log file uploaded to "+ log_file)
+
+# Sleep for a random time 
+def sleep_with_backoff(base, max):
+    sleep(random.randint(base, max))
 
 ##############################################
 # Initialize variables
 # create aws clients
 client = boto3.client('iot')
 dataClient = boto3.client('iot-data')
 deviceAdvisor = boto3.client('iotdeviceadvisor')
+s3 = boto3.resource('s3')
+
+# const
+BACKOFF_BASE = 5
+BACKOFF_MAX = 10
 
 # load test config
 f = open('deviceadvisor/script/DATestConfig.json')
@@ -159,6 +184,7 @@ def delete_thing_with_certi(thingName, certiId, certiArn):
         # 'createdAt': datetime(2015, 1, 1)
         # }
         print("[Device Advisor]Info: Start device advisor test: " + test_name)
+        sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX)
         test_start_response = deviceAdvisor.start_suite_run(
         suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
         suiteRunConfiguration={
@@ -175,8 +201,8 @@ def delete_thing_with_certi(thingName, certiId, certiArn):
         os.environ['DA_ENDPOINT'] = endpoint_response['endpoint']
 
         while True:
-            # sleep for 1s every loop to avoid TooManyRequestsException
-            sleep(1)
+            # Add backoff to avoid TooManyRequestsException
+            sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX)
             test_result_responds = deviceAdvisor.get_suite_run(
                 suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
                 suiteRunId=test_start_response['suiteRunId']
@@ -204,10 +230,18 @@ def delete_thing_with_certi(thingName, certiId, certiArn):
             # If the test finalizing or store the test result
             elif (test_result_responds['status'] != 'RUNNING'):
                 test_result[test_name] = test_result_responds['status']
-                if(test_result[test_name] == "PASS"):
-                    delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
+                # If the test failed, upload the logs to S3 before clean up
+                if(test_result[test_name] != "PASS"):
+                    log_url = test_result_responds['testResult']['groups'][0]['tests'][0]['logUrl']
+                    group_string = re.search('group=(.*);', log_url)
+                    log_group = group_string.group(1)
+                    stream_string = re.search('stream=(.*)', log_url)
+                    log_stream = stream_string.group(1)
+                    process_logs(log_group, log_stream, thing_name)
+                delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
                 break
     except Exception as e:
+        delete_thing_with_certi(thing_name, certificate_id ,certificate_arn)
         print("[Device Advisor]Error: Failed to test: "+ test_name)
 
 ##############################################