Skip to content

Decouple model e2e tests #487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Decouple model e2e tests #487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 21 additions & 17 deletions .ci/scripts/gather_test_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,25 +49,29 @@ def export_models_for_ci() -> None:
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
models = {"include": []}
for name in MODEL_NAME_TO_MODEL.keys():
quantization = (
name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].quantization
)
xnnpack_delegation = (
quantization_configs = {
False,
name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].quantization,
}
delegation_configs = {
False,
name in MODEL_NAME_TO_OPTIONS
and MODEL_NAME_TO_OPTIONS[name].xnnpack_delegation
)
and MODEL_NAME_TO_OPTIONS[name].xnnpack_delegation,
}
for build_tool in BUILD_TOOLS:
models["include"].append(
{
"build-tool": build_tool,
"model": name,
"quantization": quantization,
"xnnpack_delegation": xnnpack_delegation,
"runner": RUNNERS.get(name, DEFAULT_RUNNER),
# demo_backend_delegation test only supports add_mul model
"demo_backend_delegation": name == "add_mul",
}
)
for q_config in quantization_configs:
for d_config in delegation_configs:
models["include"].append(
{
"build-tool": build_tool,
"model": name,
"xnnpack_quantization": q_config,
"xnnpack_delegation": d_config,
"runner": RUNNERS.get(name, DEFAULT_RUNNER),
# demo_backend_delegation test only supports add_mul model
"demo_backend_delegation": name == "add_mul",
}
)
set_output("models", json.dumps(models))


Expand Down
41 changes: 18 additions & 23 deletions .ci/scripts/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ if [[ -z "${BUILD_TOOL:-}" ]]; then
exit 1
fi

QUANTIZATION=$3
if [[ -z "${QUANTIZATION:-}" ]]; then
QUANTIZATION=false
XNNPACK_QUANTIZATION=$3
if [[ -z "${XNNPACK_QUANTIZATION:-}" ]]; then
XNNPACK_QUANTIZATION=false
fi

XNNPACK_DELEGATION=$4
Expand Down Expand Up @@ -88,6 +88,15 @@ build_cmake_xnn_executor_runner() {

test_model_with_xnnpack() {
WITH_QUANTIZATION=$1
WITH_DELEGATION=$2

# Quantization-only
if [[ ${WITH_QUANTIZATION} == true ]] && [[ ${WITH_DELEGATION} == false ]]; then
bash examples/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}"
exit 0
fi

# Delegation
if [[ ${WITH_QUANTIZATION} == true ]]; then
SUFFIX="q8"
"${PYTHON_EXECUTABLE}" -m examples.backend.xnnpack_examples --model_name="${MODEL_NAME}" --delegate --quantize
Expand All @@ -97,6 +106,7 @@ test_model_with_xnnpack() {
fi

OUTPUT_MODEL_PATH="${MODEL_NAME}_xnnpack_${SUFFIX}.pte"

# Run test model
if [[ "${BUILD_TOOL}" == "buck2" ]]; then
buck2 run //examples/backend:xnn_executor_runner -- --model_path "${OUTPUT_MODEL_PATH}"
Expand Down Expand Up @@ -135,27 +145,12 @@ test_demo_backend_delegation() {
fi
}

echo "Testing ${MODEL_NAME} (fp32, quantized, xnnpack) with ${BUILD_TOOL}..."
# Test the select model without XNNPACK or quantization
test_model

# Test quantization
if [[ "${QUANTIZATION}" == true ]]; then
bash examples/quantization/test_quantize.sh "${BUILD_TOOL}" "${MODEL_NAME}"
if [[ "${XNNPACK_DELEGATION}" == false ]] && [[ "${XNNPACK_QUANTIZATION}" == false ]]; then
echo "Testing ${MODEL_NAME} with portable kernels..."
test_model
else
echo "The model ${MODEL_NAME} doesn't support quantization yet"
fi

# Test XNNPACK without quantization
if [[ "${XNNPACK_DELEGATION}" == true ]]; then
test_model_with_xnnpack false
else
echo "The model ${MODEL_NAME} doesn't support XNNPACK yet"
fi

# Test XNNPACK with quantization
if [[ "${XNNPACK_DELEGATION}" == true ]] && [[ "${QUANTIZATION}" == true ]]; then
test_model_with_xnnpack true
echo "Testing ${MODEL_NAME} with XNNPACK quantization=${XNNPACK_QUANTIZATION} delegation=${XNNPACK_DELEGATION}..."
test_model_with_xnnpack "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}"
fi

# Test demo backend delegation
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/pull.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,13 @@ jobs:

MODEL_NAME=${{ matrix.model }}
BUILD_TOOL=${{ matrix.build-tool }}
QUANTIZATION=${{ matrix.quantization }}
XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
XNNPACK_DELEGATION=${{ matrix.xnnpack_delegation }}
DEMO_BACKEND_DELEGATION=${{ matrix.demo_backend_delegation }}

PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
# Build and test Executorch
PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${QUANTIZATION}" "${XNNPACK_DELEGATION}" "${DEMO_BACKEND_DELEGATION}"
PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}" "${DEMO_BACKEND_DELEGATION}"

test-models-macos:
name: test-models-macos
Expand All @@ -83,14 +83,14 @@ jobs:

MODEL_NAME=${{ matrix.model }}
BUILD_TOOL=${{ matrix.build-tool }}
QUANTIZATION=${{ matrix.quantization }}
XNNPACK_QUANTIZATION=${{ matrix.xnnpack_quantization }}
XNNPACK_DELEGATION=${{ matrix.xnnpack_delegation }}
DEMO_BACKEND_DELEGATION=${{ matrix.demo_backend_delegation }}

# Setup MacOS dependencies as there is no Docker support on MacOS atm
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
# Build and test Executorch
PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${QUANTIZATION}" "${XNNPACK_DELEGATION}" "${DEMO_BACKEND_DELEGATION}"
PYTHON_EXECUTABLE=python bash .ci/scripts/test.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${XNNPACK_QUANTIZATION}" "${XNNPACK_DELEGATION}" "${DEMO_BACKEND_DELEGATION}"
popd

test-custom-ops-linux:
Expand Down