update smoke test

Author: jathu

.ci/scripts/gather_test_models.py

@@ -33,6 +33,7 @@
         "phi-4-mini": "linux.4xlarge.memory",
     }
 }
+EVENT_NAMES = ["pull_request", "push", "schedule"]
 
 DEFAULT_TIMEOUT = 90
 CUSTOM_TIMEOUT = {
@@ -64,7 +65,7 @@ def parse_args() -> Any:
         "-e",
         "--event",
         type=str,
-        choices=["pull_request", "push", "schedule"],
+        choices=EVENT_NAMES,
         required=True,
         help="GitHub CI Event. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#on",
     )
@@ -113,13 +114,17 @@ def model_should_run_on_target_os(model: str, target_os: str) -> bool:
     return model not in ["llava"]
 
 
-def export_models_for_ci() -> dict[str, dict]:
+def export_models_for_ci(target_os: str, event: str) -> dict[str, Any]:
     """
     This gathers all the example models that we want to test on GitHub OSS CI
     """
-    args = parse_args()
-    target_os = args.target_os
-    event = args.event
+
+    if target_os not in DEFAULT_RUNNERS:
+        print(f"unknown os '{target_os}' provided")
+        exit(1)
+    elif event not in EVENT_NAMES:
+        print(f"unknown event '{event}' provided")
+        exit(1)
 
     # This is the JSON syntax for configuration matrix used by GitHub
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
@@ -175,9 +180,14 @@ def export_models_for_ci() -> dict[str, dict]:
             record["runner"] = CUSTOM_RUNNERS[target_os][name]
 
         models["include"].append(record)
-
-    set_output("models", json.dumps(models))
+    
+    return models
 
 
 if __name__ == "__main__":
-    export_models_for_ci()
+    args = parse_args()
+    models = export_models_for_ci(
+        target_os = args.target_os,
+        event = args.event
+    )
+    set_output("models", json.dumps(models))

.ci/scripts/wheel/__init__.py

@@ -8,6 +8,15 @@
 # should typically only contain shell variable assignments. Be sure to export
 # any variables so that subprocesses will see them.
 
+# Define PYTHONPATH if it's not already defined. Needed to run the smoke test
+# python script.
+if [[ -z "${PYTHONPATH:-}" ]]; then
+  export PYTHONPATH="${GITHUB_WORKSPACE}/${REPOSITORY}"
+fi
+
+# What to build the wheel with.
+export BUILD_TOOL="cmake"
+
 # Enable pybindings so that users can execute ExecuTorch programs from python.
 export EXECUTORCH_BUILD_PYBIND=1
 
@@ -18,7 +27,3 @@ export CMAKE_ARGS="${CMAKE_ARGS:-}"
 # Link the XNNPACK backend into the pybindings runtime so that users can execute
 # ExecuTorch programs that delegate to it.
 CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_XNNPACK=ON"
-
-# When building for macOS, link additional backends into the pybindings runtime.
-CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_COREML=ON"
-CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_MPS=ON"

build/packaging/env_var_script_macos.sh renamed to .ci/scripts/wheel/envvar_base.sh

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# This file is sourced into the environment before building a pip wheel. It
+# should typically only contain shell variable assignments. Be sure to export
+# any variables so that subprocesses will see them.
+
+source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh"

.ci/scripts/wheel/envvar_linux.sh

@@ -0,0 +1,15 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# This file is sourced into the environment before building a pip wheel. It
+# should typically only contain shell variable assignments. Be sure to export
+# any variables so that subprocesses will see them.
+
+source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh"
+
+# When building for macOS, link additional backends into the pybindings runtime.
+CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_COREML=ON"
+CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_MPS=ON"

.ci/scripts/wheel/envvar_macos.sh

@@ -0,0 +1,58 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import subprocess
+import sys
+from functools import lru_cache
+
+
+@lru_cache()
+def _repository_root_dir() -> str:
+  workspace_dir = os.getenv("GITHUB_WORKSPACE")
+  if workspace_dir is None:
+    print("GITHUB_WORKSPACE is not set")
+    sys.exit(1)
+
+  repository_dir = os.getenv("REPOSITORY")
+  if repository_dir is None:
+    print("REPOSITORY is not set")
+    sys.exit(1)
+
+  return os.path.join(workspace_dir, repository_dir)
+
+# Since .ci starts with a period, we cannot import it directly.
+sys.path.append(os.path.join(_repository_root_dir(), ".ci"))
+from scripts import gather_test_models
+
+
+def run_tests(target_os: str) -> None:
+  models = gather_test_models.export_models_for_ci(
+    target_os = target_os,
+    # Event refers to the type of models that will be downloaded. "pull_request"
+    # uses higher priority and fast models.
+    event="pull_request",
+  ).get("include", [])
+
+  if len(models) == 0:
+    print("No models ran")
+    sys.exit(1)
+
+  if os.getenv("PYTHONPATH") == "true":
+    print("PYTHONPATH is not set")
+    sys.exit(1)
+
+  for model in models:
+    subprocess.run(
+        [
+          os.path.join(_repository_root_dir(), ".ci/scripts/test_model.sh"),
+          model["model"], 
+          model["build-tool"], 
+          model["backend"],
+        ],
+        env=os.environ,
+        check=True,
+    )

build/packaging/post_build_script.sh renamed to .ci/scripts/wheel/post_build_script.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import test_base
+
+if __name__ == "__main__":
+    test_base.run_tests(target_os="linux")

build/packaging/pre_build_script.sh renamed to .ci/scripts/wheel/pre_build_script.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import test_base
+
+if __name__ == "__main__":
+    test_base.run_tests(target_os="macos")

.ci/scripts/wheel/test_base.py

@@ -4,7 +4,7 @@ name: Build Linux Wheels
 on:
   pull_request:
     paths:
-      - build/packaging/**
+      - .ci/**
       - .github/workflows/build-wheels-linux.yml
   push:
     branches:
@@ -39,9 +39,9 @@ jobs:
       matrix:
         include:
           - repository: pytorch/executorch
-            pre-script: build/packaging/pre_build_script.sh
-            post-script: build/packaging/post_build_script.sh
-            smoke-test-script: build/packaging/smoke_test.py
+            pre-script: .ci/scripts/wheel/pre_build_script.sh
+            post-script: .ci/scripts/wheel/post_build_script.sh
+            smoke-test-script: .ci/scripts/wheel/test_linux.py
             package-name: executorch
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
@@ -55,7 +55,7 @@ jobs:
       # "recursive" default to do less work, and to give the buck daemon fewer
       # files to look at.
       submodules: true
-      env-var-script: build/packaging/env_var_script_linux.sh
+      env-var-script: .ci/scripts/wheel/envvar_linux.sh
       pre-script: ${{ matrix.pre-script }}
       post-script: ${{ matrix.post-script }}
       package-name: ${{ matrix.package-name }}

.ci/scripts/wheel/test_linux.py

@@ -4,7 +4,7 @@ name: Build macOS Wheels
 on:
   pull_request:
     paths:
-      - build/packaging/**
+      - .ci/**
       - .github/workflows/build-wheels-macos.yml
   push:
     branches:
@@ -39,9 +39,9 @@ jobs:
       matrix:
         include:
           - repository: pytorch/executorch
-            pre-script: build/packaging/pre_build_script.sh
-            post-script: build/packaging/post_build_script.sh
-            smoke-test-script: build/packaging/smoke_test.py
+            pre-script: .ci/scripts/wheel/pre_build_script.sh
+            post-script: .ci/scripts/wheel/post_build_script.sh
+            smoke-test-script: .ci/scripts/wheel/test_macos.py
             package-name: executorch
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
@@ -56,7 +56,7 @@ jobs:
       # files to look at.
       submodules: true
       delocate-wheel: false
-      env-var-script: build/packaging/env_var_script_macos.sh
+      env-var-script: .ci/scripts/wheel/envvar_macos.sh
       pre-script: ${{ matrix.pre-script }}
       post-script: ${{ matrix.post-script }}
       package-name: ${{ matrix.package-name }}