Add playmode (in editor mode) test in CI

.github/workflows/integration_tests.yml

@@ -18,8 +18,8 @@ on:
         default: 'macos-latest'
         required: true
       platforms:
-        description: 'CSV of Android,iOS,Windows,macOS,Linux'
-        default: 'Android,iOS,Windows,macOS,Linux'
+        description: 'CSV of Android,iOS,Windows,macOS,Linux,Playmode'
+        default: 'Android,iOS,Windows,macOS,Linux,Playmode'
         required: true
       apis:
         description: 'CSV of apis to build and test'
@@ -167,7 +167,6 @@ jobs:
             TEST_MATRIX_PARAM="--auto_diff origin/${{github.event.pull_request.head.ref}}..${MERGE_BASE}"
           fi
         fi
-
         # To feed input into the job matrix, we first need to convert to a JSON
         # list. Then we can use fromJson to define the field in the matrix for the tests job.
         echo "::set-output name=apis::$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k apis -o "${{github.event.inputs.apis}}" ${TEST_MATRIX_PARAM} )"
@@ -195,6 +194,7 @@ jobs:
   build:
     name: build-${{ matrix.unity_version }}-${{matrix.os}}-${{ matrix.platform }}
     runs-on: ${{matrix.os}}
+    if: ${{ needs.check_and_prepare.outputs.matrix_build_platform != '[]' }}
     needs: check_and_prepare
     strategy:
       fail-fast: false
@@ -393,6 +393,139 @@ jobs:
             exit 1
           fi
 
+  test_playmode:
+    name: test-${{ matrix.unity_version }}-${{matrix.os}}-Playmode
+    runs-on: ${{matrix.os}}
+    if: contains(needs.check_and_prepare.outputs.platform, 'Playmode') && !cancelled()
+    needs: check_and_prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        unity_version: ${{ fromJson(needs.check_and_prepare.outputs.matrix_unity_versions) }}
+        os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_build_os) }}
+    env:
+      # LC_ALL, LANG and U3D_PASSWORD are needed for U3D.
+      LC_ALL: en_US.UTF-8
+      LANG: en_US.UTF-8
+      U3D_PASSWORD: ""
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Unity installer (U3D)
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          gem install u3d
+          u3d available
+          # u3d available -u ${{ matrix.unity_version }} -p
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Install python deps
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          pip install -r scripts/gha/requirements.txt
+      - name: Install Unity
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          python scripts/gha/unity_installer.py --install \
+            --version ${{ matrix.unity_version }}
+      - name: Activate Unity license
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          python scripts/gha/unity_installer.py --activate_license \
+            --version ${{ matrix.unity_version }}  \
+            --username "${{ secrets.UNITY_USERNAME }}" \
+            --password "${{ secrets.UNITY_PASSWORD }}" \
+            --serial_ids "${{ secrets.SERIAL_ID }}" \
+            --logfile "testapps/activate_license.log"
+      - name: Prepare for integration tests
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
+      - name: Fetch prebuilt packaged SDK from previous run
+        uses: dawidd6/action-download-artifact@v2
+        if: ${{ github.event.inputs.test_packaged_sdk != '' }}
+        with:
+          name: 'firebase_unity_sdk.zip'
+          workflow: 'packaging.yml'
+          run_id: ${{ github.event.inputs.test_packaged_sdk }}
+
+      - name: Run Playmode (in editor mode) integration tests
+        shell: bash
+        run: |
+          if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then
+            unzip -q firebase_unity_sdk.zip -d ~/Downloads/
+          else
+            if [[ -z "${{ github.event.inputs.sdk_url }}" ]];then
+              sdk_url="https://dl.google.com/firebase/sdk/unity/firebase_unity_sdk_8.7.0.zip"
+            else
+              sdk_url=${{ github.event.inputs.sdk_url }}
+            fi
+            curl ${sdk_url} -o ~/Downloads/firebase_unity_sdk.zip
+            unzip -q ~/Downloads/firebase_unity_sdk.zip -d ~/Downloads/
+          fi
+          python scripts/gha/build_testapps.py \
+            --t ${{ needs.check_and_prepare.outputs.apis }} \
+            --u $( python scripts/gha/print_matrix_configuration.py -k version -u ${{matrix.unity_version}}) \
+            --p Playmode \
+            --ios_sdk ${{ needs.check_and_prepare.outputs.mobile_test_on }} \
+            --plugin_dir ~/Downloads/firebase_unity_sdk \
+            --output_directory "${{ github.workspace }}" \
+            --artifact_name "${{ matrix.unity_version }}-${{matrix.os}}-Playmode" \
+            --force_latest_runtime \
+            --ci
+      - name: Return Unity license
+        # Always returns true, even when job failed or canceled. But will not run when a critical failure prevents the task from running. 
+        if: always()
+        # Retry at most 3 times when Return Unity license fails.
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          shell: bash
+          command: |
+            python scripts/gha/unity_installer.py --release_license \
+              --version ${{ matrix.unity_version }} \
+              --logfile "testapps/release_license.log"
+            cat testapps/release_license.log
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f test-results-${{ matrix.unity_version }}-${{matrix.os}}-Playmode.log.json ]; then
+            # No summary was created, make a placeholder one.
+            echo "__SUMMARY_MISSING__" > test-results-${{ matrix.unity_version }}-${{matrix.os}}-Playmode.log.json
+          fi
+      - name: Upload test results artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: log-artifact
+          path: test-results-${{ matrix.unity_version }}-${{matrix.os}}-Playmode*
+          retention-days: ${{ env.artifactRetentionDays }}
+      - name: Update PR label and comment
+        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
+        shell: bash
+        run: |
+          python scripts/gha/it_workflow.py --stage progress \
+            --token ${{github.token}} \
+            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
+            --actor ${{github.actor}} \
+            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
+            --run_id ${{github.run_id}}
+      - name: Summarize test results
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          cat test-results-${{ matrix.unity_version }}-${{matrix.os}}-Playmode.log
+          if [[ "${{ job.status }}" != "success" ]]; then
+            exit 1
+          fi
 
   test_desktop:
     name: test-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop
@@ -572,10 +705,9 @@ jobs:
             exit 1
           fi
 
-
   summarize_results:
     name: "summarize-results"
-    needs: [check_and_prepare, build, test_desktop, test_mobile]
+    needs: [check_and_prepare, build, test_playmode, test_desktop, test_mobile]
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }}
     steps:

scripts/gha/build_testapps.py

@@ -94,6 +94,7 @@
 import glob
 import os
 import platform
+import stat
 import shutil
 import subprocess
 import time
@@ -145,7 +146,8 @@
   _IOS: "iOS",
   _WINDOWS: "Win64",
   _MACOS: "OSXUniversal",
-  _LINUX: "Linux64"
+  _LINUX: "Linux64",
+  _PLAYMODE: "Playmode"
 }
 
 _SUPPORTED_PLATFORMS = (
@@ -264,6 +266,13 @@
     " c can be a combination of digits and letters.")
 
 
+@attr.s(frozen=False, eq=False)
+class Test(object):
+  """Holds data related to the testing of one testapp."""
+  testapp_path = attr.ib()
+  logs = attr.ib()
+
+
 def main(argv):
   del argv  # Unused.
 
@@ -286,6 +295,7 @@ def main(argv):
   platforms = validate_platforms(FLAGS.platforms)
 
   output_root = os.path.join(root_output_dir, "testapps")
+  playmode_tests = []
   failures = []
   for version in unity_versions:
     runtime = get_runtime(version, FLAGS.force_latest_runtime)
@@ -329,14 +339,17 @@ def main(argv):
           if p == _DESKTOP:  # e.g. 'Desktop' -> 'OSXUniversal'
             p = get_desktop_platform()
           if p == _PLAYMODE:
-            perform_in_editor_tests(dir_helper)
+            logs = perform_in_editor_tests(dir_helper)
+            playmode_tests.append(Test(testapp_path=dir_helper.unity_project_dir, logs=logs))
           else:
             build_testapp(
                 dir_helper=dir_helper,
                 api_config=api_config,
                 ios_config=ios_config,
                 target=_BUILD_TARGET[p])
         except (subprocess.SubprocessError, RuntimeError) as e:
+          if p == _PLAYMODE:
+            playmode_tests.append(Test(testapp_path=dir_helper.unity_project_dir, logs=str(e)))
           failures.append(
               Failure(
                   testapp=testapp, 
@@ -350,20 +363,32 @@ def main(argv):
             logging.info(f.read())
       # Free up space by removing unneeded Unity directory.
       if FLAGS.ci:
-        shutil.rmtree(dir_helper.unity_project_dir)
+        _rm_dir_safe(dir_helper.unity_project_dir)
       else:
-        shutil.rmtree(os.path.join(dir_helper.unity_project_dir, "Library"))
+        _rm_dir_safe(os.path.join(dir_helper.unity_project_dir, "Library"))
       logging.info("END %s", build_desc)
 
-  _collect_integration_tests(config, testapps, root_output_dir, output_dir, FLAGS.artifact_name)
-
-  return _summarize_results(
-      testapps=testapps,
-      platforms=platforms,
-      versions=unity_versions,
-      failures=failures,
-      output_dir=root_output_dir, 
-      artifact_name=FLAGS.artifact_name)
+  playmode_passes = True
+  build_passes = True
+  if _PLAYMODE in platforms:
+    platforms.remove(_PLAYMODE)
+    playmode_passes = test_validation.summarize_test_results(
+      playmode_tests, 
+      test_validation.UNITY, 
+      root_output_dir, 
+      file_name="test-results-" + FLAGS.artifact_name + ".log")
+
+  if platforms:
+    _collect_integration_tests(config, testapps, root_output_dir, output_dir, FLAGS.artifact_name)
+    build_passes = _summarize_build_results(
+        testapps=testapps,
+        platforms=platforms,
+        versions=unity_versions,
+        failures=failures,
+        output_dir=root_output_dir, 
+        artifact_name=FLAGS.artifact_name)
+
+  return (playmode_passes and build_passes)
 
 
 def setup_unity_project(dir_helper, setup_options):
@@ -531,7 +556,7 @@ def perform_in_editor_tests(dir_helper, retry_on_license_check=True):
       dir_helper.unity_path,
       dir_helper.unity_project_dir,
       shared_args=["-batchmode", "-nographics", "-accept-apiupdate"])
-  log = dir_helper.make_log_path("editor_tests")
+  log = dir_helper.make_log_path("build_Playmode")
   arg_builder.set_log_file(log)
   run_args = arg_builder.get_args_for_method("InEditorRunner.EditorRun")
   dir_helper.copy_editor_script("InEditorRunner.cs")
@@ -554,7 +579,7 @@ def perform_in_editor_tests(dir_helper, retry_on_license_check=True):
   open_process.kill()
   logging.info("Finished running playmode tests")
 
-  results = test_validation.validate_results_unity(text)
+  results = test_validation.validate_results(text, test_validation.UNITY)
   if results.complete:
     if results.passes and not results.fails:  # Success
       logging.info(results.summary)
@@ -564,6 +589,8 @@ def perform_in_editor_tests(dir_helper, retry_on_license_check=True):
     raise RuntimeError(
         "Tests did not finish running. Log tail:\n" + results.summary)
 
+  return text
+
 
 def run_xcodebuild(dir_helper, ios_config, device_type):
   """Uses xcode project generated by Unity to build an iOS binary."""
@@ -619,7 +646,7 @@ def _collect_integration_tests(config, testapps, root_output_dir, output_dir, ar
   artifact_path = os.path.join(root_output_dir, testapps_artifact_dir)
   logging.info("Collecting artifacts to: %s", artifact_path)
   try:
-    shutil.rmtree(artifact_path)
+    _rm_dir_safe(artifact_path)
   except OSError as e:
     logging.warning("Failed to remove directory:\n%s", e.strerror)
 
@@ -647,7 +674,7 @@ def _collect_integration_tests_platform(config, testapps, artifact_path, testapp
         break
 
 
-def _summarize_results(testapps, platforms, versions, failures, output_dir, artifact_name):
+def _summarize_build_results(testapps, platforms, versions, failures, output_dir, artifact_name):
   """Logs a readable summary of the results of the build."""
   file_name = "build-results-" + artifact_name + ".log"
   summary = []
@@ -956,6 +983,24 @@ def update_unity_versions(version_path_map, log=logging.error):
   return valid_versions
 
 
+def _handle_readonly_file(func, path, excinfo):
+  """Function passed into shutil.rmtree to handle Access Denied error"""
+  os.chmod(path, stat.S_IWRITE)
+  func(path)  # will re-throw if a different error occurrs
+
+
+def _rm_dir_safe(directory_path):
+  """Removes directory at given path. No error if dir doesn't exist."""
+  logging.info("Deleting %s...", directory_path)
+  try:
+    shutil.rmtree(directory_path, onerror=_handle_readonly_file)
+  except OSError as e:
+    # There are two known cases where this can happen:
+    # The directory doesn't exist (FileNotFoundError)
+    # A file in the directory is open in another process (PermissionError)
+    logging.warning("Failed to remove directory:\n%s", e.strerror)
+
+
 def _fix_path(path):
   """Expands ~, normalizes slashes, and converts relative paths to absolute."""
   if not path:

scripts/gha/integration_testing/test_validation.py

@@ -321,7 +321,7 @@ def get_name(testapp_path):
   """Returns testapp api."""
   testapps = PARAMETERS["integration_tests"]["config"]["apis"].split(",")
   for testapp in testapps:
-    if testapp in testapp_path:
+    if testapp.replace("_", "") in testapp_path.lower():
       return testapp
   return testapp_path
 

scripts/gha/print_matrix_configuration.py

@@ -67,7 +67,7 @@
       }
     },
     "config": {
-      "platform": "Windows,macOS,Linux,Android,iOS",
+      "platform": "Windows,macOS,Linux,Android,iOS,Playmode",
       "apis": "analytics,auth,crashlytics,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage",
       "mobile_test_on": "real"
     }
@@ -253,8 +253,10 @@ def filter_build_platform(platform):
   platform = platform.split(",")
   build_platform = []
   build_platform.extend(filter_mobile_platform(platform))
+  # testapps from different desktop platforms are built in one job.
   desktop_platform = ','.join(list(filter(lambda p: p in platform, ["Windows", "macOS", "Linux"])))
-  build_platform.append(desktop_platform)
+  if desktop_platform:
+    build_platform.append(desktop_platform)
   return build_platform