test: auto test script supports python3

Author: ydhub

examples/bluetooth/nimble/bleprph/bleprph_test.py

@@ -17,11 +17,15 @@
 from __future__ import print_function
 import os
 import re
-import Queue
 import traceback
 import threading
 import subprocess
 
+try:
+    import Queue
+except ImportError:
+    import queue as Queue
+
 from tiny_test_fw import Utility
 import ttfw_idf
 from ble import lib_ble_client

tools/ble/lib_ble_client.py

@@ -35,8 +35,8 @@
     print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue")
     raise
 
-import lib_gatt
-import lib_gap
+from . import lib_gatt
+from . import lib_gap
 
 srv_added_old_cnt = 0
 srv_added_new_cnt = 0

tools/ci/config/assign-test.yml

@@ -14,6 +14,8 @@ assign_test:
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
     TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs"
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+    # auto_test_script is compatible with Python 3 only
+    PYTHON_VER: 3
   artifacts:
     paths:
       - components/idf_test/*/CIConfigs

tools/ci/config/target-test.yml

@@ -159,6 +159,8 @@
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
     MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    # auto_test_script is compatible with Python 3 only
+    PYTHON_VER: 3
   script:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -68,7 +68,7 @@ def format_build_log_path():
         job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
         for job_info in job_info_list:
             raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
-            build_info_list = [json.loads(line) for line in raw_data.splitlines()]
+            build_info_list = [json.loads(line) for line in raw_data.decode().splitlines()]
             for build_info in build_info_list:
                 build_info["ci_job_id"] = job_info["id"]
                 artifact_index_list.append(build_info)

tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py

@@ -127,7 +127,7 @@ def output(self):
                     "name": test_function,
                     "extra_data": self._create_extra_data(test_cases, test_function),
                     "overwrite": overwrite,
-                } for test_function, test_cases in case_by_test_function.iteritems() if test_cases
+                } for test_function, test_cases in case_by_test_function.items() if test_cases
             ],
         }
         return output_data

tools/ci/setup_python.sh

@@ -1,11 +1,15 @@
 #!/usr/bin/env bash
 
+# py2_incomp='assign_test|nvs_compatible_test|IT'
+# - auto_test_script do not support python2 anymore
+# but there are still some jobs incompatible with Python 3 (eg: example_test)
+# keep the default python interpreter as 2.7.15 until all the jobs support python3
+
 # Regexp for matching job names which are incompatible with Python 3
-# - assign_test, nvs_compatible_test, IT - auto_test_script causes the incompatibility
 # - UT_009_ - multi-device tests are not compatible
 # - UT_014_ - multi-device tests are not compatible
 # - UT_017_ - multi-device tests are not compatible
-py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_|UT_013_|UT_014_|UT_017_'
+py3_incomp='UT_009_|UT_013_|UT_014_|UT_017_'
 
 if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then
     # Use this version of the Python interpreter if it was not defined before or