Skip to content

Commit 6b05051

Browse files
kimishpatelkimishpatel
committed
Update base for Update on "Dont quantize the current token for attention"
Differential Revision: [D63497872](https://our.internmc.facebook.com/intern/diff/D63497872/) [ghstack-poisoned]
2 parents db5ccb8 + 43555d2 commit 6b05051

File tree

211 files changed

+839410
-2117
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

211 files changed

+839410
-2117
lines changed

‎.ci/docker/common/install_cache.sh

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,26 @@ set -ex
1212
# shellcheck source=/dev/null
1313
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
1414

15+
install_ubuntu() {
16+
echo "Preparing to build sccache from source"
17+
apt-get update
18+
# libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04.
19+
# Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh``
20+
apt-get install -y cargo
21+
echo "Checking out sccache repo"
22+
git clone https://github.com/mozilla/sccache -b v0.8.2
23+
24+
cd sccache
25+
echo "Building sccache"
26+
cargo build --release
27+
cp target/release/sccache /opt/cache/bin
28+
echo "Cleaning up"
29+
cd ..
30+
rm -rf sccache
31+
apt-get remove -y cargo rustc
32+
apt-get autoclean && apt-get clean
33+
}
34+
1535
install_binary() {
1636
echo "Downloading sccache binary from S3 repo"
1737
curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
@@ -22,15 +42,33 @@ mkdir -p /opt/cache/bin
2242
sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
2343
export PATH="/opt/cache/bin:$PATH"
2444

25-
# NB: Install the pre-built binary from S3 as building from source
26-
# https://github.com/pytorch/sccache has started failing mysteriously
27-
# in which sccache server couldn't start with the following error:
28-
# sccache: error: Invalid argument (os error 22)
29-
install_binary
45+
install_ubuntu
3046

3147
function write_sccache_stub() {
3248
BINARY=$1
33-
printf "#!/bin/sh\nif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then\n exec sccache %s \"\$@\"\nelse\n exec %s \"\$@\"\nfi" "$(which "${BINARY}")" "$(which "${BINARY}")" > "/opt/cache/bin/${BINARY}"
49+
if [ $1 == "gcc" ]; then
50+
# Do not call sccache recursively when dumping preprocessor argument
51+
# For some reason it's very important for the first cached nvcc invocation
52+
cat >"/opt/cache/bin/$1" <<EOF
53+
#!/bin/sh
54+
if [ "\$1" = "-E" ] || [ "\$2" = "-E" ]; then
55+
exec $(which $1) "\$@"
56+
elif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then
57+
exec sccache $(which $1) "\$@"
58+
else
59+
exec $(which $1) "\$@"
60+
fi
61+
EOF
62+
else
63+
cat >"/opt/cache/bin/$1" <<EOF
64+
#!/bin/sh
65+
if [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then
66+
exec sccache $(which $1) "\$@"
67+
else
68+
exec $(which $1) "\$@"
69+
fi
70+
EOF
71+
fi
3472
chmod a+x "/opt/cache/bin/${BINARY}"
3573
}
3674

@@ -44,7 +82,7 @@ init_sccache() {
4482

4583
# NB: This function is adopted from PyTorch core at
4684
# https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/common-build.sh
47-
as_ci_user sccache --stop-server > /dev/null 2>&1 || true
85+
as_ci_user sccache --stop-server >/dev/null 2>&1 || true
4886
rm -f "${SCCACHE_ERROR_LOG}" || true
4987

5088
# Clear sccache stats before using it

‎.ci/docker/ubuntu/Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ COPY ./common/utils.sh utils.sh
5757
RUN bash ./install_cache.sh && rm install_cache.sh utils.sh
5858
ENV SCCACHE_BUCKET ossci-compiler-cache-circleci-v2
5959
ENV SCCACHE_S3_KEY_PREFIX executorch
60+
ENV SCCACHE_REGION us-east-1
6061

6162
ARG TORCH_VERSION
6263
COPY ./common/install_pytorch.sh install_pytorch.sh

‎.ci/scripts/gather_test_models.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
"resnet50": "linux.12xlarge",
2626
"llava": "linux.12xlarge",
2727
"llama3_2_vision_encoder": "linux.12xlarge",
28-
"llama3_2_text_decoder": "linux.12xlarge",
28+
# "llama3_2_text_decoder": "linux.12xlarge", # TODO: re-enable test when Huy's change is in / model gets smaller.
2929
# This one causes timeout on smaller runner, the root cause is unclear (T161064121)
3030
"dl3": "linux.12xlarge",
3131
"emformer_join": "linux.12xlarge",

‎.ci/scripts/test_llama.sh

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,41 @@ set -exu
99
# shellcheck source=/dev/null
1010
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
1111

12-
MODEL_NAME=$1 # stories110M
13-
BUILD_TOOL=$2 # buck2 or cmake
14-
DTYPE=$3 # fp16, bf16, or fp32
15-
MODE=${4:-"xnnpack+custom"} # portable or xnnpack+custom or xnnpack+custom+qe
16-
UPLOAD_DIR=${5:-}
12+
while [[ $# -gt 0 ]]; do
13+
case "$1" in
14+
-model)
15+
MODEL_NAME="$2" # stories110M
16+
shift 2
17+
;;
18+
-build_tool)
19+
BUILD_TOOL="$2" # buck2 or cmake
20+
shift 2
21+
;;
22+
-dtype)
23+
DTYPE="$2" # fp16, bf16, or fp32
24+
shift 2
25+
;;
26+
-mode)
27+
MODE="$2" # portable or xnnpack+custom or xnnpack+custom+qe
28+
shift 2
29+
;;
30+
-upload)
31+
UPLOAD_DIR="$2"
32+
shift 2
33+
;;
34+
*)
35+
echo "Unknown option: $1"
36+
usage
37+
;;
38+
esac
39+
done
40+
41+
# Default mode to xnnpack+custom if not set
42+
MODE=${MODE:-"xnnpack+custom"}
43+
44+
# Default UPLOAD_DIR to empty string if not set
45+
UPLOAD_DIR="${UPLOAD_DIR:-}"
46+
1747
if [[ $# -lt 4 ]]; then # Assuming 4 mandatory args
1848
echo "Expecting atleast 4 positional arguments"
1949
echo "Usage: [...]"
@@ -150,7 +180,7 @@ cleanup_files() {
150180
}
151181

152182
prepare_artifacts_upload() {
153-
if [ -n "$UPLOAD_DIR" ]; then
183+
if [ -n "${UPLOAD_DIR}" ]; then
154184
echo "Preparing for uploading generated artifacs"
155185
zip -j model.zip "${EXPORTED_MODEL_NAME}" tokenizer.bin
156186
mkdir -p "${UPLOAD_DIR}"

‎.github/scripts/check_labels.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env python3
2+
"""Check whether a PR has required labels."""
3+
4+
import sys
5+
from typing import Any
6+
7+
from github_utils import gh_delete_comment, gh_post_pr_comment
8+
from gitutils import get_git_remote_name, get_git_repo_dir, GitRepo
9+
from label_utils import has_required_labels, is_label_err_comment, LABEL_ERR_MSG
10+
from trymerge import GitHubPR
11+
12+
13+
def delete_all_label_err_comments(pr: "GitHubPR") -> None:
14+
for comment in pr.get_comments():
15+
if is_label_err_comment(comment):
16+
gh_delete_comment(pr.org, pr.project, comment.database_id)
17+
18+
19+
def add_label_err_comment(pr: "GitHubPR") -> None:
20+
# Only make a comment if one doesn't exist already
21+
if not any(is_label_err_comment(comment) for comment in pr.get_comments()):
22+
gh_post_pr_comment(pr.org, pr.project, pr.pr_num, LABEL_ERR_MSG)
23+
24+
25+
def parse_args() -> Any:
26+
from argparse import ArgumentParser
27+
28+
parser = ArgumentParser("Check PR labels")
29+
parser.add_argument("pr_num", type=int)
30+
# add a flag to return a non-zero exit code if the PR does not have the required labels
31+
parser.add_argument(
32+
"--exit-non-zero",
33+
action="store_true",
34+
help="Return a non-zero exit code if the PR does not have the required labels",
35+
)
36+
37+
return parser.parse_args()
38+
39+
40+
def main() -> None:
41+
args = parse_args()
42+
repo = GitRepo(get_git_repo_dir(), get_git_remote_name())
43+
org, project = repo.gh_owner_and_name()
44+
pr = GitHubPR(org, project, args.pr_num)
45+
46+
try:
47+
if not has_required_labels(pr):
48+
print(LABEL_ERR_MSG, flush=True)
49+
add_label_err_comment(pr)
50+
if args.exit_non_zero:
51+
raise RuntimeError("PR does not have required labels")
52+
else:
53+
delete_all_label_err_comments(pr)
54+
except Exception as e:
55+
if args.exit_non_zero:
56+
raise RuntimeError(f"Error checking labels: {e}") from e
57+
58+
sys.exit(0)
59+
60+
61+
if __name__ == "__main__":
62+
main()

‎.github/scripts/github_utils.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,10 @@ def gh_fetch_url(
7272
headers: Optional[Dict[str, str]] = None,
7373
data: Union[Optional[Dict[str, Any]], str] = None,
7474
method: Optional[str] = None,
75-
reader: Callable[[Any], Any] = lambda x: x.read(),
75+
reader: Callable[[Any], Any] = json.load,
7676
) -> Any:
7777
return gh_fetch_url_and_headers(
78-
url, headers=headers, data=data, reader=json.load, method=method
78+
url, headers=headers, data=data, reader=reader, method=method
7979
)[1]
8080

8181

@@ -169,7 +169,7 @@ def gh_post_commit_comment(
169169

170170
def gh_delete_comment(org: str, repo: str, comment_id: int) -> None:
171171
url = f"{GITHUB_API_URL}/repos/{org}/{repo}/issues/comments/{comment_id}"
172-
gh_fetch_url(url, method="DELETE")
172+
gh_fetch_url(url, method="DELETE", reader=lambda x: x.read())
173173

174174

175175
def gh_fetch_merge_base(org: str, repo: str, base: str, head: str) -> str:

‎.github/workflows/android-perf.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ jobs:
136136
fail-fast: false
137137
with:
138138
runner: linux.4xlarge
139-
docker-image: executorch-ubuntu-22.04-clang12-android
139+
docker-image: executorch-ubuntu-22.04-qnn-sdk
140140
submodules: 'true'
141141
timeout: 60
142142
upload-artifact: android-models

‎.github/workflows/apple.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ jobs:
269269
git config --global user.name "PyTorch Bot"
270270
git config --global user.email "[email protected]"
271271
git add Package.swift
272-
git commit -m "${VERSION}"
272+
git commit -am "${VERSION}"
273273
git push -f origin "${BRANCH}"
274274
else
275275
echo "Draft Package.swift:"

‎.github/workflows/check-labels.yml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Check Labels
2+
3+
on:
4+
# We need pull_request_target to be able to post comments on PRs from forks.
5+
# Only allow pull_request_target when merging to main, not some historical branch.
6+
#
7+
# Make sure to don't introduce explicit checking out and installing/running
8+
# untrusted user code into this workflow!
9+
pull_request_target:
10+
types: [opened, synchronize, reopened, labeled, unlabeled]
11+
branches: [main]
12+
13+
# To check labels on ghstack PRs.
14+
# Note: as pull_request doesn't trigger on PRs targeting main,
15+
# to test changes to the workflow itself one needs to create
16+
# a PR that targets a gh/**/base branch.
17+
pull_request:
18+
types: [opened, synchronize, reopened, labeled, unlabeled]
19+
branches: [gh/**/base]
20+
21+
workflow_dispatch:
22+
inputs:
23+
pr_number:
24+
description: 'PR number to check labels for'
25+
required: true
26+
27+
concurrency:
28+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
29+
cancel-in-progress: true
30+
31+
jobs:
32+
check-labels:
33+
permissions:
34+
contents: read
35+
pull-requests: write
36+
name: Check labels
37+
if: github.repository_owner == 'pytorch'
38+
runs-on: ubuntu-22.04
39+
steps:
40+
- uses: actions/checkout@v3
41+
with:
42+
fetch-depth: 0
43+
- uses: actions/setup-python@v4
44+
with:
45+
python-version: '3.10'
46+
# Not the direct dependencies but the script uses trymerge
47+
- run: pip install pyyaml==6.0 rockset==1.0.3
48+
- name: Check labels
49+
env:
50+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
51+
PR_NUM: ${{ github.event.number || github.event.inputs.pr_number }}
52+
run: |
53+
set -ex
54+
python3 .github/scripts/check_labels.py --exit-non-zero "${PR_NUM}"

‎.github/workflows/docker-builds.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ concurrency:
2626
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
2727
cancel-in-progress: true
2828

29+
env:
30+
AWS_DEFAULT_REGION: us-east-1
31+
2932
jobs:
3033
docker-build:
3134
runs-on: [self-hosted, linux.2xlarge]

‎.github/workflows/pull.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ jobs:
117117
# Install requirements for export_llama
118118
PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
119119
# Test llama2
120-
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh stories110M "${BUILD_TOOL}" "${DTYPE}" "${MODE}" "${ARTIFACTS_DIR_NAME}"
120+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -dtype "${DTYPE}" -mode "${MODE}" -upload "${ARTIFACTS_DIR_NAME}"
121121
122122
test-llama-runner-linux-android:
123123
name: test-llama-runner-linux-android
@@ -393,7 +393,7 @@ jobs:
393393
# Install requirements for export_llama
394394
PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
395395
# Test llama2
396-
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh stories110M "${BUILD_TOOL}" "${DTYPE}" "${MODE}"
396+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -dtype "${DTYPE}" -mode "${MODE}"
397397
398398
test-phi-3-mini-runner-linux:
399399
name: test-phi-3-mini-runner-linux

‎.github/workflows/trunk.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ jobs:
261261
# Install requirements for export_llama
262262
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/models/llama/install_requirements.sh
263263
# Test llama2
264-
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_llama.sh stories110M cmake "${DTYPE}" "${MODE}"
264+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_llama.sh -model stories110M -build_tool cmake -dtype "${DTYPE}" -mode "${MODE}"
265265
266266
# # TODO(jackzhxng): Runner consistently runs out of memory before test finishes. Try to find a more powerful runner.
267267
# test-llava-runner-macos:
@@ -302,7 +302,7 @@ jobs:
302302
fail-fast: false
303303
with:
304304
runner: linux.2xlarge
305-
docker-image: executorch-ubuntu-22.04-clang12-android
305+
docker-image: executorch-ubuntu-22.04-qnn-sdk
306306
submodules: 'true'
307307
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
308308
timeout: 900

‎.gitmodules

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
[submodule "backends/arm/third-party/ethos-u-core-driver"]
22
path = backends/arm/third-party/ethos-u-core-driver
3-
url = https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver
3+
url = https://github.com/pytorch-labs/ethos-u-core-driver-mirror
44
[submodule "backends/arm/third-party/serialization_lib"]
55
path = backends/arm/third-party/serialization_lib
6-
url = https://review.mlplatform.org/tosa/serialization_lib
6+
url = https://github.com/pytorch-labs/tosa_serialization_lib-mirror
77
[submodule "backends/vulkan/third-party/Vulkan-Headers"]
88
path = backends/vulkan/third-party/Vulkan-Headers
99
url = https://github.com/KhronosGroup/Vulkan-Headers

‎backends/arm/TARGETS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ python_library(
99
typing = True,
1010
deps = [
1111
":arm_backend",
12+
"//executorch/backends/arm/operator_support:operator_support",
1213
"//executorch/backends/arm/_passes:passes",
1314
"//executorch/exir:lib",
1415
],

‎backends/arm/operator_support/TARGETS

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
2+
3+
python_library(
4+
name = "operator_support",
5+
srcs = glob(["*.py"]),
6+
typing = True,
7+
deps = [
8+
"//executorch/backends/xnnpack/_passes:xnnpack_passes",
9+
"//executorch/exir:lib",
10+
"//executorch/backends/arm:tosa_specification"
11+
],
12+
)

0 commit comments

Comments
 (0)