chendl/compositional_test

1ARG BASE_DOCKER_IMAGE2FROM $BASE_DOCKER_IMAGE3LABEL maintainer="Hugging Face"4 5ARG DEBIAN_FRONTEND=noninteractive6 7# Use login shell to read variables from `~/.profile` (to pass dynamic created variables between RUN commands)8SHELL ["sh", "-lc"]9 10RUN apt update11RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs libaio-dev12RUN git lfs install13RUN python3 -m pip install --no-cache-dir --upgrade pip14 15ARG REF=main16RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF17RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime]18 19# When installing in editable mode, `transformers` is not recognized as a package.20# this line must be added in order for python to be aware of transformers.21RUN cd transformers && python3 setup.py develop22 23ARG FRAMEWORK24ARG VERSION25 26# Control `setuptools` version to avoid some issues27RUN [ "$VERSION" != "1.9" -a "$VERSION" != "1.10" ] && python3 -m pip install -U setuptools || python3 -m pip install -U "setuptools<=59.5"28 29# Remove all frameworks30# (`accelerate` requires `torch`, and this causes import issues for TF-only testing)31RUN python3 -m pip uninstall -y torch torchvision torchaudio accelerate tensorflow jax flax32 33# Get the libraries and their versions to install, and write installation command to `~/.profile`.34RUN python3 ./transformers/utils/past_ci_versions.py --framework $FRAMEWORK --version $VERSION35 36# Install the target framework37RUN echo "INSTALL_CMD = $INSTALL_CMD"38RUN $INSTALL_CMD39 40RUN [ "$FRAMEWORK" != "pytorch" ] && echo "`deepspeed-testing` installation is skipped" || python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing]41 42# Uninstall `torch-tensorrt` and `apex` shipped with the base image43RUN python3 -m pip uninstall -y torch-tensorrt apex44 45# Pre-build **nightly** release of DeepSpeed, so it would be ready for testing (otherwise, the 1st deepspeed test will timeout)46RUN python3 -m pip uninstall -y deepspeed47# This has to be run inside the GPU VMs running the tests. (So far, it fails here due to GPU checks during compilation.)48# Issue: https://github.com/microsoft/DeepSpeed/issues/201049# RUN git clone https://github.com/microsoft/DeepSpeed && cd DeepSpeed && rm -rf build && \50#    DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 python3 -m pip install . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&151 52RUN python3 -m pip install -U "itsdangerous<2.1.0"53 54# When installing in editable mode, `transformers` is not recognized as a package.55# this line must be added in order for python to be aware of transformers.56RUN cd transformers && python3 setup.py develop57