again

Author: calderjo

Dockerfile.tmpl

@@ -5,7 +5,6 @@ ARG GPU_BASE_IMAGE_NAME
 ARG LIGHTGBM_VERSION
 ARG TORCH_VERSION
 ARG TORCHAUDIO_VERSION
-ARG TORCHTEXT_VERSION
 ARG TORCHVISION_VERSION
 ARG JAX_VERSION
 
@@ -47,7 +46,6 @@ ENV TENSORFLOW_IO_VERSION=0.37.0
 ARG LIGHTGBM_VERSION
 ARG TORCH_VERSION
 ARG TORCHAUDIO_VERSION
-ARG TORCHTEXT_VERSION
 ARG TORCHVISION_VERSION
 ARG JAX_VERSION
 
@@ -62,7 +60,6 @@ ENV KMP_SETTINGS=false
 ENV PIP_ROOT_USER_ACTION=ignore
 
 ADD clean-layer.sh  /tmp/clean-layer.sh
-ADD patches/keras_patch.sh  /tmp/keras_patch.sh
 ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl
 ADD patches/template_conf.json /opt/kaggle/conf.json
 
@@ -89,6 +86,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list &
     apt-get install -y build-essential unzip cmake libboost-dev libboost-system-dev libboost-filesystem-dev p7zip-full && \
     # b/182601974: ssh client was removed from the base image but is required for packages such as stable-baselines.
     apt-get install -y openssh-client && \
+    apt-get install -y graphviz && pip install graphviz && \
     /tmp/clean-layer.sh
 
 # b/128333086: Set PROJ_LIB to points to the proj4 cartographic library.
@@ -125,7 +123,7 @@ RUN pip install spacy && \
 COPY --from=torch_whl /tmp/whl/*.whl /tmp/torch/
 RUN mamba install -y -c pytorch magma-cuda${CUDA_MAJOR_VERSION}${CUDA_MINOR_VERSION} && \
     pip install /tmp/torch/*.whl && \
-    # b/255757999 openmp (libomp.so) is an dependency of libtorchtext and libtorchaudio but
+    # b/255757999 openmp (libomp.so) is an dependency libtorchaudio but
     mamba install -y openmp && \
     rm -rf /tmp/torch && \
     /tmp/clean-layer.sh
@@ -134,8 +132,7 @@ RUN pip install \
         torch==$TORCH_VERSION+cpu \
         torchvision==$TORCHVISION_VERSION+cpu \
         torchaudio==$TORCHAUDIO_VERSION+cpu \
-        torchtext==$TORCHTEXT_VERSION \
-    -f https://download.pytorch.org/whl/torch_stable.html && \
+        --index-url https://download.pytorch.org/whl/cpu && \
     /tmp/clean-layer.sh
 {{ end }}
 
@@ -203,15 +200,13 @@ RUN pip install \
         "tensorflow-io==${TENSORFLOW_IO_VERSION}" \
         tensorflow-probability \
         tensorflow_decision_forests \
+        tensorflow-text \
         "tensorflow_hub>=0.16.0" \
         # b/331799280 remove once other packages over to dm-tre
         optree \
         tf-keras && \
     /tmp/clean-layer.sh
 
-RUN chmod +x /tmp/keras_patch.sh && \
-    /tmp/keras_patch.sh
-
 ADD patches/keras_internal.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal.py
 ADD patches/keras_internal_test.py /opt/conda/lib/python3.10/site-packages/tensorflow_decision_forests/keras/keras_internal_test.py
 
@@ -221,12 +216,13 @@ RUN pip install "keras>3" keras-cv keras-nlp && \
 # b/328788268 libpysal 4.10 seems to fail with "module 'shapely' has no attribute 'Geometry'. Did you mean: 'geometry'"
 RUN pip install pysal "libpysal==4.9.2"
 
+# b/350573866 xgboost v2.1.0 breaks learntools
 RUN apt-get install -y libfreetype6-dev && \
     apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \
     pip install gensim \
         textblob \
         wordcloud \
-        xgboost \
+        "xgboost==2.0.3" \
         pydot \
         hep_ml && \
     # NLTK Project datasets
@@ -272,7 +268,6 @@ RUN pip install "scipy==1.12.0" \
         datashader \
         # Boruta (python implementation)
         Boruta && \
-    apt-get install -y graphviz && pip install graphviz && \
     # Pandoc is a dependency of deap
     apt-get install -y pandoc && \
     pip install essentia
@@ -471,8 +466,8 @@ RUN pip install bleach \
         widgetsnbextension \
         # Require pyarrow newer than https://github.com/advisories/GHSA-5wvp-7f3h-6wmm
         {{ if eq .Accelerator "gpu" }} pyarrow {{ else }} "pyarrow>=14.0.1" {{ end }} \
-        feather-format \
-        fastai
+        feather-format && \
+    pip install --no-deps fastai fastcore fastdownload
 
 RUN python -m spacy download en_core_web_sm && python -m spacy download en_core_web_lg && \
     apt-get update && apt-get install -y ffmpeg && \

Jenkinsfile

@@ -36,7 +36,6 @@ pipeline {
                 --package torch \
                 --version $TORCH_VERSION \
                 --build-arg TORCHAUDIO_VERSION=$TORCHAUDIO_VERSION \
-                --build-arg TORCHTEXT_VERSION=$TORCHTEXT_VERSION \
                 --build-arg TORCHVISION_VERSION=$TORCHVISION_VERSION \
                 --build-arg CUDA_MAJOR_VERSION=$CUDA_MAJOR_VERSION \
                 --build-arg CUDA_MINOR_VERSION=$CUDA_MINOR_VERSION \

config.txt

@@ -5,7 +5,6 @@ GPU_BASE_IMAGE_NAME=tf2-gpu.2-16.py310
 LIGHTGBM_VERSION=4.2.0
 TORCH_VERSION=2.4.0
 TORCHAUDIO_VERSION=2.4.0
-TORCHTEXT_VERSION=0.18.0
 TORCHVISION_VERSION=0.19.0
 JAX_VERSION=0.4.26
 CUDA_MAJOR_VERSION=12

packages/torch.Dockerfile

@@ -4,7 +4,6 @@ FROM ${BASE_IMAGE} AS builder
 
 ARG PACKAGE_VERSION
 ARG TORCHAUDIO_VERSION
-ARG TORCHTEXT_VERSION
 ARG TORCHVISION_VERSION
 ARG CUDA_MAJOR_VERSION
 ARG CUDA_MINOR_VERSION
@@ -63,18 +62,6 @@ RUN sudo apt-get update && \
 RUN sed -i 's/set(envs/set(envs\n  "LIBS=-ltinfo"/' /usr/local/src/audio/third_party/sox/CMakeLists.txt 
 RUN cd /usr/local/src/audio && python setup.py bdist_wheel
 
-# Build torchtext
-# Instructions: https://github.com/pytorch/text#building-from-source
-# See comment above for PYTORCH_BUILD_VERSION.
-ENV BUILD_VERSION=$TORCHTEXT_VERSION
-RUN cd /usr/local/src && \
-    git clone https://github.com/pytorch/text && \
-    cd text && \
-    git checkout tags/v$TORCHTEXT_VERSION && \
-    git submodule sync && \
-    git submodule update --init --recursive --jobs 1 && \
-    python setup.py bdist_wheel
-
 # Build torchvision.
 # Instructions: https://github.com/pytorch/vision/tree/main#installation
 # See comment above for PYTORCH_BUILD_VERSION.
@@ -93,7 +80,6 @@ FROM alpine:latest
 RUN mkdir -p /tmp/whl/
 COPY --from=builder /usr/local/src/pytorch/dist/*.whl /tmp/whl
 COPY --from=builder /usr/local/src/audio/dist/*.whl /tmp/whl
-COPY --from=builder /usr/local/src/text/dist/*.whl /tmp/whl
 COPY --from=builder /usr/local/src/vision/dist/*.whl /tmp/whl
 
 # Print out the built .whl file.

patches/keras_patch.sh

@@ -1,16 +1,11 @@
 import unittest
 
 import geopandas
+from shapely.geometry import Polygon
 
 class TestGeopandas(unittest.TestCase):
-    def test_read(self):
-        df = geopandas.read_file(geopandas.datasets.get_path('nybb'))
-        self.assertTrue(df.size > 1)
-
-    def test_spatial_join(self):
-        cities = geopandas.read_file(geopandas.datasets.get_path('naturalearth_cities'))
-        world = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
-        countries = world[['geometry', 'name']]
-        countries = countries.rename(columns={'name':'country'})
-        cities_with_country = geopandas.sjoin(cities, countries, how="inner", op='intersects')
-        self.assertTrue(cities_with_country.size > 1)
+    def test_GeoSeries(self):
+        p1 = Polygon([(0, 0), (1, 0), (1, 1)])
+        p2 = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
+        p3 = Polygon([(2, 0), (3, 0), (3, 1), (2, 1)])
+        g = geopandas.GeoSeries([p1, p2, p3])

tests/test_geopandas.py

@@ -21,6 +21,6 @@ def test_classifier(self):
         xgb1.fit(
             X_train, y_train,
             eval_set=[(X_train, y_train), (X_test, y_test)],
-            eval_metric='mlogloss',
+            eval_metric='mlogloss'
         )
         self.assertIn("validation_0", xgb1.evals_result())

tests/test_torchtext.py

@@ -13,7 +13,6 @@ ARG TENSORFLOW_VERSION
 ARG TF_LIBTPU_VERSION
 ARG JAX_VERSION
 ARG TORCHVISION_VERSION
-ARG TORCHTEXT_VERSION
 ARG TORCHAUDIO_VERSION
 
 ENV ISTPUVM=1
@@ -60,7 +59,7 @@ RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y
 # Additional useful packages should be added here
 
 RUN pip install tensorflow_hub https://storage.googleapis.com/cloud-tpu-tpuvm-artifacts/tensorflow/tf-${TENSORFLOW_VERSION}/tensorflow-${TENSORFLOW_VERSION}-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TF_LINUX_WHEEL_VERSION}.whl tensorflow-probability tensorflow-io \
-    torch~=${TORCH_VERSION} https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}+libtpu-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl torchvision==${TORCHVISION_VERSION} torchtext==${TORCHTEXT_VERSION} torchaudio==${TORCHAUDIO_VERSION} \
+    torch~=${TORCH_VERSION} https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}+libtpu-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl torchvision==${TORCHVISION_VERSION} torchaudio==${TORCHAUDIO_VERSION} \
     jax[tpu]==${JAX_VERSION} -f https://storage.googleapis.com/jax-releases/libtpu_releases.html trax flax optax git+https://github.com/deepmind/dm-haiku jraph distrax \
     papermill jupyterlab python-lsp-server[all] "jupyter-lsp==1.5.1" \
     pandas matplotlib opencv-python-headless librosa accelerate diffusers scikit-learn transformers \

tests/test_xgboost.py

@@ -12,8 +12,6 @@ JAX_VERSION=0.4.23
 TORCH_VERSION=2.3.0
 # https://github.com/pytorch/audio supports nightly
 TORCHAUDIO_VERSION=2.3.0
-# https://github.com/pytorch/text supports main
-TORCHTEXT_VERSION=0.18.0
 # https://github.com/pytorch/vision supports nightly
 TORCHVISION_VERSION=0.18.0
 TORCH_LINUX_WHEEL_VERSION=manylinux_2_28_x86_64