Skip to content

Commit 789c521

Browse files
swolchokswolchok
committed
Update on "Use c10 version of half/bfloat16 in executorch"
Accomplished by importing relevant files from c10 into executorch/runtime/core/portable_type/c10, and then using `using` in the top-level ExecuTorch headers. This approach should keep the ExecuTorch build hermetic for embedded use cases. In the future, we should add a CI job to ensure the c10 files stay identical to the PyTorch ones. Differential Revision: [D66106969](https://our.internmc.facebook.com/intern/diff/D66106969/) [ghstack-poisoned]
2 parents abe03ea + 7093c30 commit 789c521

File tree

95 files changed

+2579
-1293
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

95 files changed

+2579
-1293
lines changed

‎.github/pytorch-probot.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# The schema is from https://github.com/pytorch/pytorch/blob/main/.github/pytorch-probot.yml
2+
tracking_issue: 7679
23
ciflow_push_tags:
34
- ciflow/android
45
- ciflow/apple

‎.github/workflows/android-perf.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ jobs:
260260
--output_name="${OUT_ET_MODEL_NAME}.pte"
261261
ls -lh "${OUT_ET_MODEL_NAME}.pte"
262262
elif [[ ${{ matrix.config }} == "llama3_qnn_htp" ]]; then
263-
export QNN_SDK_ROOT=/tmp/qnn/2.25.0.240728
263+
export QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029
264264
export LD_LIBRARY_PATH=$QNN_SDK_ROOT/lib/x86_64-linux-clang/
265265
export PYTHONPATH=$(pwd)/..
266266
@@ -347,7 +347,7 @@ jobs:
347347
PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
348348
349349
export ANDROID_ABIS="arm64-v8a"
350-
PYTHON_EXECUTABLE=python EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.25.0.240728 bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME}
350+
PYTHON_EXECUTABLE=python EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029 bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME}
351351
352352
# Let's see how expensive this job is, we might want to tone it down by running it periodically
353353
benchmark-on-device:

‎.github/workflows/pull.yml

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -199,9 +199,6 @@ jobs:
199199
200200
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake"
201201
202-
# install pybind
203-
bash install_requirements.sh --pybind xnnpack
204-
205202
# install Llava requirements
206203
bash examples/models/llama/install_requirements.sh
207204
bash examples/models/llava/install_requirements.sh
@@ -333,6 +330,9 @@ jobs:
333330

334331
unittest-arm:
335332
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
333+
permissions:
334+
id-token: write
335+
contents: read
336336
with:
337337
runner: linux.2xlarge
338338
docker-image: executorch-ubuntu-22.04-arm-sdk
@@ -395,6 +395,25 @@ jobs:
395395
# Test llama2
396396
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -mode "${MODE}" -dtype "${DTYPE}" -pt2e_quantize "${PT2E_QUANTIZE}"
397397
398+
test-qnn-models-linux:
399+
name: test-qnn-models-linux
400+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
401+
strategy:
402+
fail-fast: false
403+
with:
404+
runner: linux.2xlarge
405+
docker-image: executorch-ubuntu-22.04-qnn-sdk
406+
submodules: 'true'
407+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
408+
timeout: 180
409+
script: |
410+
# The generic Linux job chooses to use base env, not the one setup by the image
411+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
412+
conda activate "${CONDA_ENV}"
413+
414+
# placeholder for running test_qnn_delegate.py, can use matrix such that we can trigger different jobs, refers to test-llama-runner-qnn-linux
415+
# reminder: make sure each job runs fast
416+
398417
test-phi-3-mini-runner-linux:
399418
name: test-phi-3-mini-runner-linux
400419
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
@@ -413,9 +432,6 @@ jobs:
413432
414433
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake"
415434
416-
# install pybind
417-
bash install_requirements.sh --pybind xnnpack
418-
419435
# install phi-3-mini requirements
420436
bash examples/models/phi-3-mini/install_requirements.sh
421437
@@ -440,9 +456,6 @@ jobs:
440456
441457
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake"
442458
443-
# install pybind
444-
bash install_requirements.sh --pybind xnnpack
445-
446459
# install llama requirements
447460
bash examples/models/llama/install_requirements.sh
448461
@@ -467,10 +480,6 @@ jobs:
467480
468481
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake"
469482
470-
# install pybind
471-
bash install_requirements.sh --pybind xnnpack
472-
473-
# install llama requirements
474483
bash examples/models/llama/install_requirements.sh
475484
476485
# run eval_llama mmlu task
@@ -494,9 +503,6 @@ jobs:
494503
495504
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake"
496505
497-
# install pybind
498-
bash install_requirements.sh --pybind xnnpack
499-
500506
# install llama requirements
501507
bash examples/models/llama/install_requirements.sh
502508

‎.github/workflows/trunk.yml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,9 @@ jobs:
132132
test-arm-backend-delegation:
133133
name: test-arm-backend-delegation
134134
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
135+
permissions:
136+
id-token: write
137+
contents: read
135138
with:
136139
runner: linux.2xlarge
137140
docker-image: executorch-ubuntu-22.04-arm-sdk
@@ -144,7 +147,7 @@ jobs:
144147
conda activate "${CONDA_ENV}"
145148
146149
source .ci/scripts/utils.sh
147-
install_executorch
150+
install_executorch "use-pt-pinned-commit"
148151
149152
.ci/scripts/setup-arm-baremetal-tools.sh
150153
@@ -159,6 +162,9 @@ jobs:
159162
test-arm-reference-delegation:
160163
name: test-arm-reference-delegation
161164
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
165+
permissions:
166+
id-token: write
167+
contents: read
162168
with:
163169
runner: linux.2xlarge
164170
docker-image: executorch-ubuntu-22.04-arm-sdk
@@ -171,7 +177,7 @@ jobs:
171177
conda activate "${CONDA_ENV}"
172178
173179
source .ci/scripts/utils.sh
174-
install_executorch
180+
install_executorch "use-pt-pinned-commit"
175181
176182
.ci/scripts/setup-arm-baremetal-tools.sh
177183

‎.lintrunner.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,7 @@ include_patterns = [
298298
'build/**/*.py',
299299
'codegen/**/*.py',
300300
# 'devtools/**/*.py',
301+
'devtools/visualization/**/*.py',
301302
'docs/**/*.py',
302303
# 'examples/**/*.py',
303304
# 'exir/**/*.py',

‎backends/apple/coreml/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,14 +93,14 @@ class Model(torch.nn.Module):
9393
source_model = Model()
9494
example_inputs = (torch.randn((1, 3, 256, 256)), )
9595

96-
pre_autograd_aten_dialect = export_for_training(model, example_inputs).module()
96+
pre_autograd_aten_dialect = export_for_training(source_model, example_inputs).module()
9797

9898
quantization_config = LinearQuantizerConfig.from_dict(
9999
{
100100
"global_config": {
101101
"quantization_scheme": QuantizationScheme.symmetric,
102-
"activation_dtype": torch.uint8,
103-
"weight_dtype": torch.int8,
102+
"activation_dtype": torch.quint8,
103+
"weight_dtype": torch.qint8,
104104
"weight_per_channel": True,
105105
}
106106
}

‎backends/arm/_passes/arm_pass_manager.py

Lines changed: 63 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
# pyre-unsafe
99

10-
import torch
1110
from executorch.backends.arm._passes.annotate_channels_last_dim_order_pass import (
1211
AnnotateChannelsLastDimOrder,
1312
)
@@ -47,7 +46,7 @@
4746
)
4847
from executorch.backends.arm._passes.match_arg_ranks_pass import MatchArgRanksPass
4948
from executorch.backends.arm._passes.meandim_to_averagepool_pass import (
50-
ConvertMeanDimToAveragePool,
49+
ConvertMeanDimToAveragePoolPass,
5150
)
5251
from executorch.backends.arm._passes.mm_to_bmm_pass import ConvertMmToBmmPass
5352
from executorch.backends.arm._passes.remove_clone_pass import RemoveClonePass
@@ -61,86 +60,98 @@
6160
from executorch.backends.arm._passes.unsqueeze_scalar_placeholders_pass import (
6261
UnsqueezeScalarPlaceholdersPass,
6362
)
63+
from executorch.backends.arm.tosa_specification import TosaSpecification
6464
from executorch.backends.xnnpack._passes.remove_getitem_op import RemoveGetItemPass
6565
from executorch.exir import ExportedProgram
66-
from executorch.exir.dialects._ops import ops as exir_ops
6766
from executorch.exir.pass_manager import PassManager
67+
from torch.fx import GraphModule
6868

6969

7070
class ArmPassManager(PassManager):
7171

72-
def _transform(self, graph_module: torch.fx.GraphModule):
72+
def __init__(self, tosa_spec: TosaSpecification) -> None:
73+
self.tosa_spec = tosa_spec
74+
super().__init__()
75+
76+
def _transform(self, graph_module: GraphModule):
7377
return self(graph_module).graph_module
7478

75-
def transform_to_backend_pipeline(self, exported_program: ExportedProgram):
76-
"""Apply passes before transforming program to backend"""
79+
def _tosa_080_BI_pipeline(self, exported_program: ExportedProgram) -> GraphModule:
7780
self.add_pass(FuseQuantizedActivationPass())
81+
self.add_pass(RemoveGetItemPass())
82+
self.add_pass(ConvertSplitToSlicePass())
83+
self.add_pass(ConvertMmToBmmPass())
7884
self.add_pass(DecomposeLinearPass())
85+
self.add_pass(ConvertMeanDimToAveragePoolPass())
86+
87+
self.add_pass(AnnotateDecomposedMatmulPass())
88+
self.add_pass(QuantizeFullArgument())
89+
self.add_pass(FoldAndAnnotateQParamsPass())
90+
self.add_pass(RetraceFoldedDtypesPass())
91+
self.add_pass(InsertTableOpsPass(exported_program))
92+
93+
self.add_pass(RemoveClonePass())
94+
self.add_pass(SizeAdjustConv2DPass())
95+
self.add_pass(ConvertExpandCopyToRepeatPass())
96+
self.add_pass(UnsqueezeBeforeRepeatPass())
97+
self.add_pass(UnsqueezeScalarPlaceholdersPass(exported_program))
98+
self.add_pass(CastInt64ToInt32Pass(exported_program))
99+
self.add_pass(MatchArgRanksPass(exported_program))
100+
self.add_pass(KeepDimsFalseToSqueezePass())
101+
self.add_pass(Conv1dUnsqueezePass(exported_program))
102+
self.add_pass(DecomposeSelectPass())
103+
104+
self.add_pass(AnnotateChannelsLastDimOrder())
105+
106+
return self._transform(exported_program.graph_module)
107+
108+
def _tosa_080_MI_pipeline(self, exported_program: ExportedProgram) -> GraphModule:
109+
110+
self.add_pass(FuseQuantizedActivationPass())
79111
self.add_pass(RemoveGetItemPass())
112+
self.add_pass(ConvertSplitToSlicePass())
113+
self.add_pass(ConvertMmToBmmPass())
114+
self.add_pass(DecomposeLinearPass())
80115
self.add_pass(DecomposeLayerNormPass())
81116
self.add_pass(DecomposeVarPass())
82-
self.add_pass(ConvertMeanDimToAveragePool())
83117
self.add_pass(DecomposeMeanDimPass())
84-
self.add_pass(ConvertSplitToSlicePass())
85-
self.add_pass(ConvertMmToBmmPass())
86-
# TODO MLETORCH-558
118+
self.add_pass(ConvertMeanDimToAveragePoolPass())
119+
self.add_pass(DecomposeDivPass())
120+
self.add_pass(DecomposeSoftmaxesPass())
121+
87122
self.add_pass(AnnotateDecomposedMatmulPass())
88123
self.add_pass(QuantizeFullArgument())
89-
self.add_pass(
90-
FoldAndAnnotateQParamsPass(
91-
[
92-
exir_ops.edge.aten.minimum.default,
93-
exir_ops.edge.aten.maximum.default,
94-
exir_ops.edge.aten.add.Tensor,
95-
exir_ops.edge.aten.avg_pool2d.default,
96-
exir_ops.edge.aten.bmm.default,
97-
exir_ops.edge.aten.cat.default,
98-
exir_ops.edge.aten.convolution.default,
99-
exir_ops.edge.aten.clone.default,
100-
exir_ops.edge.aten.exp.default,
101-
exir_ops.edge.aten.expand_copy.default,
102-
exir_ops.edge.aten.full.default,
103-
exir_ops.edge.aten.hardtanh.default,
104-
exir_ops.edge.aten.log.default,
105-
exir_ops.edge.aten.max_pool2d.default,
106-
exir_ops.edge.aten.mul.Tensor,
107-
exir_ops.edge.aten.permute_copy.default,
108-
exir_ops.edge.aten.reciprocal.default,
109-
exir_ops.edge.aten.relu.default,
110-
exir_ops.edge.aten.repeat.default,
111-
exir_ops.edge.aten.rsqrt.default,
112-
exir_ops.edge.aten.select_copy.int,
113-
exir_ops.edge.aten.sigmoid.default,
114-
exir_ops.edge.aten.slice_copy.Tensor,
115-
exir_ops.edge.aten.squeeze_copy.dims,
116-
exir_ops.edge.aten.sub.Tensor,
117-
exir_ops.edge.aten.sum.dim_IntList,
118-
exir_ops.edge.aten.tanh.default,
119-
exir_ops.edge.aten.unsqueeze_copy.default,
120-
exir_ops.edge.aten.upsample_nearest2d.vec,
121-
exir_ops.edge.aten.view_copy.default,
122-
]
123-
)
124-
)
124+
self.add_pass(FoldAndAnnotateQParamsPass())
125125
self.add_pass(RetraceFoldedDtypesPass())
126126
self.add_pass(InsertTableOpsPass(exported_program))
127+
128+
self.add_pass(RemoveClonePass())
129+
self.add_pass(SizeAdjustConv2DPass())
127130
self.add_pass(ConvertExpandCopyToRepeatPass())
128131
self.add_pass(UnsqueezeBeforeRepeatPass())
129-
self.add_pass(CastInt64ToInt32Pass(exported_program))
130132
self.add_pass(UnsqueezeScalarPlaceholdersPass(exported_program))
131-
self.add_pass(SizeAdjustConv2DPass())
132-
self.add_pass(RemoveClonePass())
133+
self.add_pass(CastInt64ToInt32Pass(exported_program))
133134
self.add_pass(MatchArgRanksPass(exported_program))
134-
self.add_pass(DecomposeDivPass())
135135
self.add_pass(KeepDimsFalseToSqueezePass())
136136
self.add_pass(Conv1dUnsqueezePass(exported_program))
137-
self.add_pass(DecomposeSoftmaxesPass())
138137
self.add_pass(DecomposeSelectPass())
138+
139139
self.add_pass(AnnotateChannelsLastDimOrder())
140140

141141
return self._transform(exported_program.graph_module)
142142

143-
def transform_for_annotation_pipeline(self, graph_module: torch.fx.GraphModule):
143+
def transform_to_backend_pipeline(self, exported_program: ExportedProgram):
144+
"""Apply passes before transforming program to backend"""
145+
if self.tosa_spec == TosaSpecification.create_from_string("TOSA-0.80.0+BI"):
146+
return self._tosa_080_BI_pipeline(exported_program)
147+
elif self.tosa_spec == TosaSpecification.create_from_string("TOSA-0.80.0+MI"):
148+
return self._tosa_080_MI_pipeline(exported_program)
149+
else:
150+
raise NotImplementedError(
151+
f"No pass pipeline implemented for {self.tosa_spec=}"
152+
)
153+
154+
def transform_for_annotation_pipeline(self, graph_module: GraphModule):
144155
self.add_pass(ScalarsToAttributePass())
145156
self.add_pass(DecomposeLayerNormPass())
146157
self.add_pass(DecomposeVarPass())

‎backends/arm/_passes/cast_int64_pass.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2024 Arm Limited and/or its affiliates.
1+
# Copyright 2024-2025 Arm Limited and/or its affiliates.
22
#
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
@@ -17,6 +17,10 @@
1717

1818

1919
class CastInt64ToInt32Pass(ExportPass):
20+
"""
21+
Cast int64 buffers to int32 if the int64 data is in int32 range.
22+
"""
23+
2024
def __init__(self, exported_program: torch.export.ExportedProgram):
2125
super(CastInt64ToInt32Pass, self).__init__()
2226
self.exported_program = exported_program

0 commit comments

Comments
 (0)