Skip to content

Commit 9dad297

Browse files
jithunnair-amdjithunnair-amd
authored
Merge pull request #981 from ROCmSoftwarePlatform/IFU-master-2022-03-30
IFU-master-2022-03-30
2 parents 5536ac2 + 396fad6 commit 9dad297

File tree

683 files changed

+33025
-48655
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

683 files changed

+33025
-48655
lines changed

‎.circleci/docker/build.sh

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,21 +222,21 @@ case "$image" in
222222
DB=yes
223223
VISION=yes
224224
;;
225-
pytorch-linux-bionic-rocm4.3.1-py3.7)
225+
pytorch-linux-bionic-rocm4.5-py3.7)
226226
ANACONDA_PYTHON_VERSION=3.7
227227
GCC_VERSION=9
228228
PROTOBUF=yes
229229
DB=yes
230230
VISION=yes
231-
ROCM_VERSION=4.3.1
231+
ROCM_VERSION=4.5.2
232232
;;
233-
pytorch-linux-bionic-rocm4.5-py3.7)
233+
pytorch-linux-bionic-rocm5.0-py3.7)
234234
ANACONDA_PYTHON_VERSION=3.7
235235
GCC_VERSION=9
236236
PROTOBUF=yes
237237
DB=yes
238238
VISION=yes
239-
ROCM_VERSION=4.5.2
239+
ROCM_VERSION=5.0
240240
;;
241241
*)
242242
# Catch-all for builds that are not hardcoded.
@@ -283,6 +283,13 @@ fi
283283

284284
tmp_tag=$(basename "$(mktemp -u)" | tr '[:upper:]' '[:lower:]')
285285

286+
#when using cudnn version 8 install it separately from cuda
287+
if [[ "$image" == *cuda* && ${OS} == "ubuntu" ]]; then
288+
IMAGE_NAME="nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
289+
if [[ ${CUDNN_VERSION} == 8 ]]; then
290+
IMAGE_NAME="nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
291+
fi
292+
fi
286293

287294
# Build image
288295
# TODO: build-arg THRIFT is not turned on for any image, remove it once we confirm
@@ -321,6 +328,7 @@ docker build \
321328
--build-arg "KATEX=${KATEX:-}" \
322329
--build-arg "ROCM_VERSION=${ROCM_VERSION:-}" \
323330
--build-arg "PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH:-gfx900;gfx906}" \
331+
--build-arg "IMAGE_NAME=${IMAGE_NAME}" \
324332
-f $(dirname ${DOCKERFILE})/Dockerfile \
325333
-t "$tmp_tag" \
326334
"$@" \

‎.circleci/docker/centos-rocm/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@ RUN bash ./install_user.sh && rm install_user.sh
4242
# Install conda and other packages (e.g., numpy, pytest)
4343
ENV PATH /opt/conda/bin:$PATH
4444
ARG ANACONDA_PYTHON_VERSION
45+
ADD requirements-ci.txt /opt/conda/requirements-ci.txt
4546
ADD ./common/install_conda.sh install_conda.sh
4647
RUN bash ./install_conda.sh && rm install_conda.sh
48+
RUN rm /opt/conda/requirements-ci.txt
4749

4850
# (optional) Install protobuf for ONNX
4951
ARG PROTOBUF

‎.circleci/docker/common/install_conda.sh

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
2121
;;
2222
esac
2323

24-
mkdir /opt/conda
24+
mkdir -p /opt/conda
2525
chown jenkins:jenkins /opt/conda
2626

2727
# Work around bug where devtoolset replaces sudo and breaks it.
@@ -94,21 +94,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
9494
conda_install nnpack -c killeent
9595

9696
# Install some other packages, including those needed for Python test reporting
97-
# Pin SciPy because of failing distribution tests (see #60347)
98-
# Pin MyPy version because new errors are likely to appear with each release
99-
# Pin hypothesis to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
100-
# Pin unittest-xml-reporting to freeze printing test summary logic, related: https://github.com/pytorch/pytorch/issues/69014
101-
as_jenkins pip install --progress-bar off pytest \
102-
scipy==1.6.3 \
103-
scikit-image \
104-
psutil \
105-
"unittest-xml-reporting<=3.2.0,>=2.0.0" \
106-
boto3==1.16.34 \
107-
hypothesis==4.53.2 \
108-
expecttest==0.1.3 \
109-
mypy==0.812 \
110-
tb-nightly \
111-
librosa>=0.6.2
97+
as_jenkins pip install --progress-bar off -r /opt/conda/requirements-ci.txt
11298

11399
# Install numba only on python-3.8 or below
114100
# For numba issue see https://github.com/pytorch/pytorch/issues/51511

‎.circleci/docker/common/install_cudnn.sh

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
if [[ ${CUDNN_VERSION} == 8 ]]; then
4+
# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
5+
mkdir tmp_cudnn && cd tmp_cudnn
6+
CUDNN_NAME="cudnn-linux-x86_64-8.3.2.44_cuda11.5-archive"
7+
curl -OLs https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.2/local_installers/11.5/${CUDNN_NAME}.tar.xz
8+
tar xf ${CUDNN_NAME}.tar.xz
9+
cp -a ${CUDNN_NAME}/include/* /usr/include/
10+
cp -a ${CUDNN_NAME}/include/* /usr/local/cuda/include/
11+
cp -a ${CUDNN_NAME}/include/* /usr/include/x86_64-linux-gnu/
12+
13+
cp -a ${CUDNN_NAME}/lib/* /usr/local/cuda/lib64/
14+
cp -a ${CUDNN_NAME}/lib/* /usr/lib/x86_64-linux-gnu/
15+
cd ..
16+
rm -rf tmp_cudnn
17+
ldconfig
18+
fi

‎.circleci/docker/common/install_rocm.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ install_magma() {
66
# "install" hipMAGMA into /opt/rocm/magma by copying after build
77
git clone https://bitbucket.org/icl/magma.git
88
pushd magma
9-
# Mar 7 - Fixes memory leaks for many linalg UTs
9+
# Fixes memory leaks of magma found while executing linalg UTs
1010
git checkout 5959b8783e45f1809812ed96ae762f38ee701972
1111
cp make.inc-examples/make.inc.hip-gcc-mkl make.inc
1212
echo 'LIBDIR += -L$(MKLROOT)/lib' >> make.inc
@@ -35,7 +35,7 @@ ver() {
3535
}
3636

3737
# Map ROCm version to AMDGPU version
38-
declare -A AMDGPU_VERSIONS=( ["4.5.2"]="21.40.2" )
38+
declare -A AMDGPU_VERSIONS=( ["4.5.2"]="21.40.2" ["5.0"]="21.50" )
3939

4040
install_ubuntu() {
4141
apt-get update

‎.circleci/docker/requirements-ci.txt

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
# Python dependencies required for unit tests
2+
3+
#awscli==1.6 #this breaks some platforms
4+
#Description: AWS command line interface
5+
#Pinned versions: 1.6
6+
#test that import:
7+
8+
boto3==1.19.12
9+
#Description: AWS SDK for python
10+
#Pinned versions: 1.19.12, 1.16.34
11+
#test that import:
12+
13+
click
14+
#Description: Command Line Interface Creation Kit
15+
#Pinned versions:
16+
#test that import:
17+
18+
coremltools==5.0b5
19+
#Description: Apple framework for ML integration
20+
#Pinned versions: 5.0b5
21+
#test that import:
22+
23+
#dataclasses #this breaks some platforms
24+
#Description: Provides decorators for auto adding special methods to user classes
25+
#Pinned versions:
26+
#test that import:
27+
28+
expecttest==0.1.3
29+
#Description: method for writing tests where test framework auto populates
30+
# the expected output based on previous runs
31+
#Pinned versions: 0.1.3
32+
#test that import:
33+
34+
flatbuffers==2.0
35+
#Description: cross platform serialization library
36+
#Pinned versions: 2.0
37+
#test that import:
38+
39+
#future #this breaks linux-bionic-rocm4.5-py3.7
40+
#Description: compatibility layer between python 2 and python 3
41+
#Pinned versions:
42+
#test that import:
43+
44+
hypothesis==4.53.2
45+
# Pin hypothesis to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136
46+
#Description: advanced library for generating parametrized tests
47+
#Pinned versions: 3.44.6, 4.53.2
48+
#test that import: test_xnnpack_integration.py, test_pruning_op.py, test_nn.py
49+
50+
junitparser==2.1.1
51+
#Description: unitparser handles JUnit/xUnit Result XML files
52+
#Pinned versions: 2.1.1
53+
#test that import:
54+
55+
librosa>=0.6.2
56+
#Description: A python package for music and audio analysis
57+
#Pinned versions: >=0.6.2
58+
#test that import: test_spectral_ops.py
59+
60+
#mkl #this breaks linux-bionic-rocm4.5-py3.7
61+
#Description: Intel oneAPI Math Kernel Library
62+
#Pinned versions:
63+
#test that import: test_profiler.py, test_public_bindings.py, test_testing.py,
64+
#test_nn.py, test_mkldnn.py, test_jit.py, test_fx_experimental.py,
65+
#test_autograd.py
66+
67+
#mkl-devel
68+
# see mkl
69+
70+
#mock # breaks ci/circleci: docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
71+
#Description: A testing library that allows you to replace parts of your
72+
#system under test with mock objects
73+
#Pinned versions:
74+
#test that import: test_module_init.py, test_modules.py, test_nn.py,
75+
#test_testing.py
76+
77+
#MonkeyType # breaks pytorch-xla-linux-bionic-py3.7-clang8
78+
#Description: collects runtime types of function arguments and return
79+
#values, and can automatically generate stub files
80+
#Pinned versions:
81+
#test that import:
82+
83+
mypy==0.812
84+
# Pin MyPy version because new errors are likely to appear with each release
85+
#Description: linter
86+
#Pinned versions: 0.812
87+
#test that import: test_typing.py, test_type_hints.py
88+
89+
#networkx
90+
#Description: creation, manipulation, and study of
91+
#the structure, dynamics, and functions of complex networks
92+
#Pinned versions: 2.0
93+
#test that import:
94+
95+
#ninja
96+
#Description: build system. Note that it install from
97+
#here breaks things so it is commented out
98+
#Pinned versions: 1.10.0.post1
99+
#test that import: run_test.py, test_cpp_extensions_aot.py,test_determination.py
100+
101+
#numba
102+
#Description: Just-In-Time Compiler for Numerical Functions
103+
#Pinned versions: 0.54.1, 0.49.0, <=0.49.1
104+
#test that import: test_numba_integration.py
105+
106+
#numpy
107+
#Description: Provides N-dimensional arrays and linear algebra
108+
#Pinned versions: 1.20
109+
#test that import: test_view_ops.py, test_unary_ufuncs.py, test_type_promotion.py,
110+
#test_type_info.py, test_torch.py, test_tensorexpr_pybind.py, test_tensorexpr.py,
111+
#test_tensorboard.py, test_tensor_creation_ops.py, test_static_runtime.py,
112+
#test_spectral_ops.py, test_sort_and_select.py, test_shape_ops.py,
113+
#test_segment_reductions.py, test_reductions.py, test_pruning_op.py,
114+
#test_overrides.py, test_numpy_interop.py, test_numba_integration.py
115+
#test_nn.py, test_namedtensor.py, test_linalg.py, test_jit_cuda_fuser.py,
116+
#test_jit.py, test_indexing.py, test_datapipe.py, test_dataloader.py,
117+
#test_binary_ufuncs.py
118+
119+
#onnxruntime
120+
#Description: scoring engine for Open Neural Network Exchange (ONNX) models
121+
#Pinned versions: 1.9.0
122+
#test that import:
123+
124+
#pillow
125+
#Description: Python Imaging Library fork
126+
#Pinned versions:
127+
#test that import:
128+
129+
#protobuf
130+
#Description: Google’s data interchange format
131+
#Pinned versions:
132+
#test that import: test_tensorboard.py
133+
134+
psutil
135+
#Description: information on running processes and system utilization
136+
#Pinned versions:
137+
#test that import: test_profiler.py, test_openmp.py, test_dataloader.py
138+
139+
pytest
140+
#Description: testing framework
141+
#Pinned versions:
142+
#test that import: test_typing.py, test_cpp_extensions_aot.py, run_test.py
143+
144+
#pytest-benchmark
145+
#Description: fixture for benchmarking code
146+
#Pinned versions: 3.2.3
147+
#test that import:
148+
149+
#pytest-sugar
150+
#Description: shows failures and errors instantly
151+
#Pinned versions:
152+
#test that import:
153+
154+
#PyYAML
155+
#Description: data serialization format
156+
#Pinned versions:
157+
#test that import:
158+
159+
#requests
160+
#Description: HTTP library
161+
#Pinned versions:
162+
#test that import: test_type_promotion.py
163+
164+
#rich
165+
#Description: rich text and beautiful formatting in the terminal
166+
#Pinned versions: 10.9.0
167+
#test that import:
168+
169+
scikit-image
170+
#Description: image processing routines
171+
#Pinned versions:
172+
#test that import: test_nn.py
173+
174+
#scikit-learn
175+
#Description: machine learning package
176+
#Pinned versions: 0.20.3
177+
#test that import:
178+
179+
scipy==1.6.3
180+
# Pin SciPy because of failing distribution tests (see #60347)
181+
#Description: scientific python
182+
#Pinned versions: 1.6.3
183+
#test that import: test_unary_ufuncs.py, test_torch.py,test_tensor_creation_ops.py
184+
#test_spectral_ops.py, test_sparse_csr.py, test_reductions.py,test_nn.py
185+
#test_linalg.py, test_binary_ufuncs.py
186+
187+
#tabulate
188+
#Description: Pretty-print tabular data
189+
#Pinned versions:
190+
#test that import:
191+
192+
tb-nightly
193+
#Description: TensorBoard
194+
#Pinned versions:
195+
#test that import:
196+
197+
#typing-extensions
198+
#Description: type hints for python
199+
#Pinned versions:
200+
#test that import:
201+
202+
#virtualenv
203+
#Description: virtual environment for python
204+
#Pinned versions:
205+
#test that import:
206+
207+
unittest-xml-reporting<=3.2.0,>=2.0.0
208+
#Description: saves unit test results to xml
209+
#Pinned versions:
210+
#test that import:

‎.circleci/docker/ubuntu-cuda/Dockerfile

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
ARG UBUNTU_VERSION
22
ARG CUDA_VERSION
3-
ARG CUDNN_VERSION
3+
ARG IMAGE_NAME
44

5-
FROM nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu${UBUNTU_VERSION}
5+
FROM ${IMAGE_NAME}
66

77
ARG UBUNTU_VERSION
88
ARG CUDA_VERSION
9-
ARG CUDNN_VERSION
109

1110
ENV DEBIAN_FRONTEND noninteractive
1211

@@ -27,8 +26,10 @@ RUN bash ./install_katex.sh && rm install_katex.sh
2726
# Install conda and other packages (e.g., numpy, pytest)
2827
ENV PATH /opt/conda/bin:$PATH
2928
ARG ANACONDA_PYTHON_VERSION
29+
ADD requirements-ci.txt /opt/conda/requirements-ci.txt
3030
ADD ./common/install_conda.sh install_conda.sh
3131
RUN bash ./install_conda.sh && rm install_conda.sh
32+
RUN rm /opt/conda/requirements-ci.txt
3233

3334
# Install gcc
3435
ARG GCC_VERSION
@@ -99,5 +100,11 @@ ENV CUDA_PATH /usr/local/cuda
99100
# Install LLVM dev version (Defined in the pytorch/builder github repository)
100101
COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
101102

103+
# Install CUDNN
104+
ARG CUDNN_VERSION
105+
ADD ./common/install_cudnn.sh install_cudnn.sh
106+
RUN if [ "${CUDNN_VERSION}" -eq 8 ]; then bash install_cudnn.sh; fi
107+
RUN rm install_cudnn.sh
108+
102109
USER jenkins
103110
CMD ["bash"]

‎.circleci/docker/ubuntu-rocm/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@ RUN bash ./install_user.sh && rm install_user.sh
2828
# Install conda and other packages (e.g., numpy, pytest)
2929
ENV PATH /opt/conda/bin:$PATH
3030
ARG ANACONDA_PYTHON_VERSION
31+
ADD requirements-ci.txt /opt/conda/requirements-ci.txt
3132
ADD ./common/install_conda.sh install_conda.sh
3233
RUN bash ./install_conda.sh && rm install_conda.sh
34+
RUN rm /opt/conda/requirements-ci.txt
3335

3436
# Install gcc
3537
ARG GCC_VERSION

‎.circleci/docker/ubuntu/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,10 @@ RUN bash ./install_katex.sh && rm install_katex.sh
3636
# Install conda and other packages (e.g., numpy, pytest)
3737
ENV PATH /opt/conda/bin:$PATH
3838
ARG ANACONDA_PYTHON_VERSION
39+
ADD requirements-ci.txt /opt/conda/requirements-ci.txt
3940
ADD ./common/install_conda.sh install_conda.sh
4041
RUN bash ./install_conda.sh && rm install_conda.sh
42+
RUN rm /opt/conda/requirements-ci.txt
4143

4244
# Install gcc
4345
ARG GCC_VERSION

0 commit comments

Comments
 (0)