Skip to content

Commit b0d0a77

Browse files
helunwencserhelunwencser
committed
Update base for Update on "Transform model to be able to use Attention Sink"
This PR adds necessary functions for transforming the model to be able to use Attention Sink. Differential Revision: [D65571289](https://our.internmc.facebook.com/intern/diff/D65571289/) [ghstack-poisoned]
2 parents 552f624 + fbcc9a1 commit b0d0a77

File tree

248 files changed

+15489
-3118
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

248 files changed

+15489
-3118
lines changed

‎.ci/docker/common/install_cache.sh

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,26 @@ set -ex
1212
# shellcheck source=/dev/null
1313
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
1414

15+
install_ubuntu() {
16+
echo "Preparing to build sccache from source"
17+
apt-get update
18+
# libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04.
19+
# Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh``
20+
apt-get install -y cargo
21+
echo "Checking out sccache repo"
22+
git clone https://github.com/mozilla/sccache -b v0.8.2
23+
24+
cd sccache
25+
echo "Building sccache"
26+
cargo build --release
27+
cp target/release/sccache /opt/cache/bin
28+
echo "Cleaning up"
29+
cd ..
30+
rm -rf sccache
31+
apt-get remove -y cargo rustc
32+
apt-get autoclean && apt-get clean
33+
}
34+
1535
install_binary() {
1636
echo "Downloading sccache binary from S3 repo"
1737
curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
@@ -22,15 +42,33 @@ mkdir -p /opt/cache/bin
2242
sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
2343
export PATH="/opt/cache/bin:$PATH"
2444

25-
# NB: Install the pre-built binary from S3 as building from source
26-
# https://github.com/pytorch/sccache has started failing mysteriously
27-
# in which sccache server couldn't start with the following error:
28-
# sccache: error: Invalid argument (os error 22)
29-
install_binary
45+
install_ubuntu
3046

3147
function write_sccache_stub() {
3248
BINARY=$1
33-
printf "#!/bin/sh\nif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then\n exec sccache %s \"\$@\"\nelse\n exec %s \"\$@\"\nfi" "$(which "${BINARY}")" "$(which "${BINARY}")" > "/opt/cache/bin/${BINARY}"
49+
if [ $1 == "gcc" ]; then
50+
# Do not call sccache recursively when dumping preprocessor argument
51+
# For some reason it's very important for the first cached nvcc invocation
52+
cat >"/opt/cache/bin/$1" <<EOF
53+
#!/bin/sh
54+
if [ "\$1" = "-E" ] || [ "\$2" = "-E" ]; then
55+
exec $(which $1) "\$@"
56+
elif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then
57+
exec sccache $(which $1) "\$@"
58+
else
59+
exec $(which $1) "\$@"
60+
fi
61+
EOF
62+
else
63+
cat >"/opt/cache/bin/$1" <<EOF
64+
#!/bin/sh
65+
if [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then
66+
exec sccache $(which $1) "\$@"
67+
else
68+
exec $(which $1) "\$@"
69+
fi
70+
EOF
71+
fi
3472
chmod a+x "/opt/cache/bin/${BINARY}"
3573
}
3674

@@ -44,7 +82,7 @@ init_sccache() {
4482

4583
# NB: This function is adopted from PyTorch core at
4684
# https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/common-build.sh
47-
as_ci_user sccache --stop-server > /dev/null 2>&1 || true
85+
as_ci_user sccache --stop-server >/dev/null 2>&1 || true
4886
rm -f "${SCCACHE_ERROR_LOG}" || true
4987

5088
# Clear sccache stats before using it

‎.ci/docker/ubuntu/Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ COPY ./common/utils.sh utils.sh
5757
RUN bash ./install_cache.sh && rm install_cache.sh utils.sh
5858
ENV SCCACHE_BUCKET ossci-compiler-cache-circleci-v2
5959
ENV SCCACHE_S3_KEY_PREFIX executorch
60+
ENV SCCACHE_REGION us-east-1
6061

6162
ARG TORCH_VERSION
6263
COPY ./common/install_pytorch.sh install_pytorch.sh

‎.ci/scripts/gather_test_models.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
"resnet50": "linux.12xlarge",
2626
"llava": "linux.12xlarge",
2727
"llama3_2_vision_encoder": "linux.12xlarge",
28-
"llama3_2_text_decoder": "linux.12xlarge",
28+
# "llama3_2_text_decoder": "linux.12xlarge", # TODO: re-enable test when Huy's change is in / model gets smaller.
2929
# This one causes timeout on smaller runner, the root cause is unclear (T161064121)
3030
"dl3": "linux.12xlarge",
3131
"emformer_join": "linux.12xlarge",

‎.ci/scripts/test_llama.sh

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,41 @@ set -exu
99
# shellcheck source=/dev/null
1010
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
1111

12-
MODEL_NAME=$1 # stories110M
13-
BUILD_TOOL=$2 # buck2 or cmake
14-
DTYPE=$3 # fp16, bf16, or fp32
15-
MODE=${4:-"xnnpack+custom"} # portable or xnnpack+custom or xnnpack+custom+qe
16-
UPLOAD_DIR=${5:-}
12+
while [[ $# -gt 0 ]]; do
13+
case "$1" in
14+
-model)
15+
MODEL_NAME="$2" # stories110M
16+
shift 2
17+
;;
18+
-build_tool)
19+
BUILD_TOOL="$2" # buck2 or cmake
20+
shift 2
21+
;;
22+
-dtype)
23+
DTYPE="$2" # fp16, bf16, or fp32
24+
shift 2
25+
;;
26+
-mode)
27+
MODE="$2" # portable or xnnpack+custom or xnnpack+custom+qe
28+
shift 2
29+
;;
30+
-upload)
31+
UPLOAD_DIR="$2"
32+
shift 2
33+
;;
34+
*)
35+
echo "Unknown option: $1"
36+
usage
37+
;;
38+
esac
39+
done
40+
41+
# Default mode to xnnpack+custom if not set
42+
MODE=${MODE:-"xnnpack+custom"}
43+
44+
# Default UPLOAD_DIR to empty string if not set
45+
UPLOAD_DIR="${UPLOAD_DIR:-}"
46+
1747
if [[ $# -lt 4 ]]; then # Assuming 4 mandatory args
1848
echo "Expecting atleast 4 positional arguments"
1949
echo "Usage: [...]"
@@ -150,7 +180,7 @@ cleanup_files() {
150180
}
151181

152182
prepare_artifacts_upload() {
153-
if [ -n "$UPLOAD_DIR" ]; then
183+
if [ -n "${UPLOAD_DIR}" ]; then
154184
echo "Preparing for uploading generated artifacs"
155185
zip -j model.zip "${EXPORTED_MODEL_NAME}" tokenizer.bin
156186
mkdir -p "${UPLOAD_DIR}"

‎.github/scripts/check_labels.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env python3
2+
"""Check whether a PR has required labels."""
3+
4+
import sys
5+
from typing import Any
6+
7+
from github_utils import gh_delete_comment, gh_post_pr_comment
8+
from gitutils import get_git_remote_name, get_git_repo_dir, GitRepo
9+
from label_utils import has_required_labels, is_label_err_comment, LABEL_ERR_MSG
10+
from trymerge import GitHubPR
11+
12+
13+
def delete_all_label_err_comments(pr: "GitHubPR") -> None:
14+
for comment in pr.get_comments():
15+
if is_label_err_comment(comment):
16+
gh_delete_comment(pr.org, pr.project, comment.database_id)
17+
18+
19+
def add_label_err_comment(pr: "GitHubPR") -> None:
20+
# Only make a comment if one doesn't exist already
21+
if not any(is_label_err_comment(comment) for comment in pr.get_comments()):
22+
gh_post_pr_comment(pr.org, pr.project, pr.pr_num, LABEL_ERR_MSG)
23+
24+
25+
def parse_args() -> Any:
26+
from argparse import ArgumentParser
27+
28+
parser = ArgumentParser("Check PR labels")
29+
parser.add_argument("pr_num", type=int)
30+
# add a flag to return a non-zero exit code if the PR does not have the required labels
31+
parser.add_argument(
32+
"--exit-non-zero",
33+
action="store_true",
34+
help="Return a non-zero exit code if the PR does not have the required labels",
35+
)
36+
37+
return parser.parse_args()
38+
39+
40+
def main() -> None:
41+
args = parse_args()
42+
repo = GitRepo(get_git_repo_dir(), get_git_remote_name())
43+
org, project = repo.gh_owner_and_name()
44+
pr = GitHubPR(org, project, args.pr_num)
45+
46+
try:
47+
if not has_required_labels(pr):
48+
print(LABEL_ERR_MSG, flush=True)
49+
add_label_err_comment(pr)
50+
if args.exit_non_zero:
51+
raise RuntimeError("PR does not have required labels")
52+
else:
53+
delete_all_label_err_comments(pr)
54+
except Exception as e:
55+
if args.exit_non_zero:
56+
raise RuntimeError(f"Error checking labels: {e}") from e
57+
58+
sys.exit(0)
59+
60+
61+
if __name__ == "__main__":
62+
main()

‎.github/scripts/github_utils.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,10 @@ def gh_fetch_url(
7272
headers: Optional[Dict[str, str]] = None,
7373
data: Union[Optional[Dict[str, Any]], str] = None,
7474
method: Optional[str] = None,
75-
reader: Callable[[Any], Any] = lambda x: x.read(),
75+
reader: Callable[[Any], Any] = json.load,
7676
) -> Any:
7777
return gh_fetch_url_and_headers(
78-
url, headers=headers, data=data, reader=json.load, method=method
78+
url, headers=headers, data=data, reader=reader, method=method
7979
)[1]
8080

8181

@@ -169,7 +169,7 @@ def gh_post_commit_comment(
169169

170170
def gh_delete_comment(org: str, repo: str, comment_id: int) -> None:
171171
url = f"{GITHUB_API_URL}/repos/{org}/{repo}/issues/comments/{comment_id}"
172-
gh_fetch_url(url, method="DELETE")
172+
gh_fetch_url(url, method="DELETE", reader=lambda x: x.read())
173173

174174

175175
def gh_fetch_merge_base(org: str, repo: str, base: str, head: str) -> str:

‎.github/workflows/android-perf.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ jobs:
136136
fail-fast: false
137137
with:
138138
runner: linux.4xlarge
139-
docker-image: executorch-ubuntu-22.04-clang12-android
139+
docker-image: executorch-ubuntu-22.04-qnn-sdk
140140
submodules: 'true'
141141
timeout: 60
142142
upload-artifact: android-models

‎.github/workflows/check-labels.yml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Check Labels
2+
3+
on:
4+
# We need pull_request_target to be able to post comments on PRs from forks.
5+
# Only allow pull_request_target when merging to main, not some historical branch.
6+
#
7+
# Make sure to don't introduce explicit checking out and installing/running
8+
# untrusted user code into this workflow!
9+
pull_request_target:
10+
types: [opened, synchronize, reopened, labeled, unlabeled]
11+
branches: [main]
12+
13+
# To check labels on ghstack PRs.
14+
# Note: as pull_request doesn't trigger on PRs targeting main,
15+
# to test changes to the workflow itself one needs to create
16+
# a PR that targets a gh/**/base branch.
17+
pull_request:
18+
types: [opened, synchronize, reopened, labeled, unlabeled]
19+
branches: [gh/**/base]
20+
21+
workflow_dispatch:
22+
inputs:
23+
pr_number:
24+
description: 'PR number to check labels for'
25+
required: true
26+
27+
concurrency:
28+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
29+
cancel-in-progress: true
30+
31+
jobs:
32+
check-labels:
33+
permissions:
34+
contents: read
35+
pull-requests: write
36+
name: Check labels
37+
if: github.repository_owner == 'pytorch'
38+
runs-on: ubuntu-22.04
39+
steps:
40+
- uses: actions/checkout@v3
41+
with:
42+
fetch-depth: 0
43+
- uses: actions/setup-python@v4
44+
with:
45+
python-version: '3.10'
46+
# Not the direct dependencies but the script uses trymerge
47+
- run: pip install pyyaml==6.0 rockset==1.0.3
48+
- name: Check labels
49+
env:
50+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
51+
PR_NUM: ${{ github.event.number || github.event.inputs.pr_number }}
52+
run: |
53+
set -ex
54+
python3 .github/scripts/check_labels.py --exit-non-zero "${PR_NUM}"

‎.github/workflows/docker-builds.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ concurrency:
2626
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
2727
cancel-in-progress: true
2828

29+
env:
30+
AWS_DEFAULT_REGION: us-east-1
31+
2932
jobs:
3033
docker-build:
3134
runs-on: [self-hosted, linux.2xlarge]

‎.github/workflows/pull.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ jobs:
117117
# Install requirements for export_llama
118118
PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
119119
# Test llama2
120-
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh stories110M "${BUILD_TOOL}" "${DTYPE}" "${MODE}" "${ARTIFACTS_DIR_NAME}"
120+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -dtype "${DTYPE}" -mode "${MODE}" -upload "${ARTIFACTS_DIR_NAME}"
121121
122122
test-llama-runner-linux-android:
123123
name: test-llama-runner-linux-android
@@ -393,7 +393,7 @@ jobs:
393393
# Install requirements for export_llama
394394
PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
395395
# Test llama2
396-
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh stories110M "${BUILD_TOOL}" "${DTYPE}" "${MODE}"
396+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -dtype "${DTYPE}" -mode "${MODE}"
397397
398398
test-phi-3-mini-runner-linux:
399399
name: test-phi-3-mini-runner-linux

‎.github/workflows/trunk.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ jobs:
261261
# Install requirements for export_llama
262262
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/models/llama/install_requirements.sh
263263
# Test llama2
264-
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_llama.sh stories110M cmake "${DTYPE}" "${MODE}"
264+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_llama.sh -model stories110M -build_tool cmake -dtype "${DTYPE}" -mode "${MODE}"
265265
266266
# # TODO(jackzhxng): Runner consistently runs out of memory before test finishes. Try to find a more powerful runner.
267267
# test-llava-runner-macos:
@@ -302,7 +302,7 @@ jobs:
302302
fail-fast: false
303303
with:
304304
runner: linux.2xlarge
305-
docker-image: executorch-ubuntu-22.04-clang12-android
305+
docker-image: executorch-ubuntu-22.04-qnn-sdk
306306
submodules: 'true'
307307
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
308308
timeout: 900

‎.gitmodules

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
[submodule "backends/arm/third-party/ethos-u-core-driver"]
22
path = backends/arm/third-party/ethos-u-core-driver
3-
url = https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver
3+
url = https://github.com/pytorch-labs/ethos-u-core-driver-mirror
44
[submodule "backends/arm/third-party/serialization_lib"]
55
path = backends/arm/third-party/serialization_lib
6-
url = https://review.mlplatform.org/tosa/serialization_lib
6+
url = https://github.com/pytorch-labs/tosa_serialization_lib-mirror
77
[submodule "backends/vulkan/third-party/Vulkan-Headers"]
88
path = backends/vulkan/third-party/Vulkan-Headers
99
url = https://github.com/KhronosGroup/Vulkan-Headers

‎CONTRIBUTING.md

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -283,10 +283,15 @@ for basics.
283283
- If the reviewers have requests or questions, follow up with them.
284284
- The goal of the reviewer is to ensure that the code in the `main` branch of
285285
the repo is consistent, maintainable, and of high quality.
286-
1. Once approved, your reviewer will import the PR into Meta's internal system
287-
and merge it from there.
288-
- If the PR is approved and not merged within a few business days, please
289-
comment on the PR to ask about its status.
286+
1. Once the PR has been approved,
287+
- If you have the "write permission" in this repo, you can merge it yourself
288+
by clicking the "Squash and merge" button once it is green and all CI
289+
signals are passing.
290+
- If you don't have "write permission" in this repo, the reviewer will take
291+
care of the PR. The reviewer may import the PR into Meta's internal system
292+
to validate it against internal CI.
293+
- If the PR is approved but not merged within 5 business days, please comment
294+
on the PR to ask about its status.
290295
- Note that if the `main` [CI](#continuous-integration) jobs are broken, we
291296
will only merge PRs that fix the broken jobs until all critical jobs are
292297
fixed.

0 commit comments

Comments
 (0)