Use PEP517 everywhere

Signed-off-by: Enrico Minack <[email protected]>
horovod · Jan 10, 2024 · a1e2fe0 · a1e2fe0
1 parent e562df2
commit a1e2fe0
Show file tree

Hide file tree

Showing 8 changed files with 13 additions and 22 deletions.
diff --git a/.github/gen-workflow-ci.py b/.github/gen-workflow-ci.py
@@ -191,9 +191,7 @@ def jobs(*jobs: str) -> str:
  ' HOROVOD_WITHOUT_MPI: 1\n' \
  ' run: |\n' \
  ' python -m pip install --upgrade pip\n' \
- ' python -m pip install setuptools wheel\n' \
- ' python setup.py sdist\n' \
- ' pip -v install dist/horovod-*.tar.gz\n' \
+ ' pip -v install --use-pep517 .\n' \
  '\n' + \
  '\n'.join(jobs)
 
@@ -480,7 +478,7 @@ def build_and_test_macos(id: str, name: str, needs: List[str], attempts: int = 3
  f' if [[ ${{TENSORFLOW}} == 1.* ]] || [[ ${{TENSORFLOW}} == 2.[012345].* ]]; then pip install "h5py<3" "protobuf~=3.20"; fi\n'
  f' pip install torch==${{PYTORCH}} pytorch_lightning==${{PYTORCH_LIGHTNING}} torchvision==${{TORCHVISION}}\n'
  f' pip install mxnet==${{MXNET}}\n'
- f' HOROVOD_WITH_TENSORFLOW=${{TENSORFLOW}} HOROVOD_WITH_PYTORCH=${{PYTORCH}} HOROVOD_WITH_MXNET=${{MXNET}} pip install --no-cache-dir .[test]\n'
+ f' HOROVOD_WITH_TENSORFLOW=${{TENSORFLOW}} HOROVOD_WITH_PYTORCH=${{PYTORCH}} HOROVOD_WITH_MXNET=${{MXNET}} pip install --no-cache-dir --use-pep517 .[test]\n'
  f' horovodrun --check-build\n'
  f'\n' +
  '\n'.join([f' - name: Test [attempt {attempt} of {attempts}]\n'

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -69,9 +69,7 @@ jobs:
  HOROVOD_WITHOUT_MPI: 1
  run: |
  python -m pip install --upgrade pip
- python -m pip install setuptools wheel
- python setup.py sdist
- pip -v install dist/horovod-*.tar.gz
+ pip -v install --use-pep517 .
 
  init-workflow:
  name: "Init Workflow"
@@ -4499,7 +4497,7 @@ jobs:
  if [[ ${TENSORFLOW} == 1.* ]] || [[ ${TENSORFLOW} == 2.[012345].* ]]; then pip install "h5py<3" "protobuf~=3.20"; fi
  pip install torch==${PYTORCH} pytorch_lightning==${PYTORCH_LIGHTNING} torchvision==${TORCHVISION}
  pip install mxnet==${MXNET}
- HOROVOD_WITH_TENSORFLOW=${TENSORFLOW} HOROVOD_WITH_PYTORCH=${PYTORCH} HOROVOD_WITH_MXNET=${MXNET} pip install --no-cache-dir .[test]
+ HOROVOD_WITH_TENSORFLOW=${TENSORFLOW} HOROVOD_WITH_PYTORCH=${PYTORCH} HOROVOD_WITH_MXNET=${MXNET} pip install --no-cache-dir --use-pep517 .[test]
  horovodrun --check-build
 
  - name: Test [attempt 1 of 3]

diff --git a/Dockerfile.test.cpu b/Dockerfile.test.cpu
@@ -234,7 +234,7 @@ RUN if [[ ${MPI_KIND} == "ONECCL" ]]; then \
  export I_MPI_ROOT=/usr/local/oneccl; \
  echo "horovod python setup.py sdist, mpicxx is $(which mpicxx)"; \
  fi; \
- bash -c "${HOROVOD_BUILD_FLAGS} HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_PACKAGE} HOROVOD_WITH_PYTORCH=${PYTORCH_PACKAGE} HOROVOD_WITH_MXNET=${MXNET_PACKAGE} pip install --no-cache-dir -v $(ls /horovod/dist/horovod-*.tar.gz)[spark,ray]"
+ bash -c "${HOROVOD_BUILD_FLAGS} HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_PACKAGE} HOROVOD_WITH_PYTORCH=${PYTORCH_PACKAGE} HOROVOD_WITH_MXNET=${MXNET_PACKAGE} pip install --no-cache-dir --use-pep517 -v /horovod[spark,ray]"
 
 # Show the effective python package version to easily spot version differences
 RUN pip freeze | sort

diff --git a/Dockerfile.test.gpu b/Dockerfile.test.gpu
@@ -212,7 +212,7 @@ RUN if [[ ${MXNET_PACKAGE} == "mxnet-nightly-cu"* ]]; then \
 
 # Install Horovod.
 RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
- bash -c "${HOROVOD_BUILD_FLAGS} HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_PACKAGE} HOROVOD_WITH_PYTORCH=${PYTORCH_PACKAGE} HOROVOD_WITH_MXNET=${MXNET_PACKAGE} pip install --no-cache-dir -v $(ls /horovod/dist/horovod-*.tar.gz)[spark,ray]" && \
+ bash -c "${HOROVOD_BUILD_FLAGS} HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_PACKAGE} HOROVOD_WITH_PYTORCH=${PYTORCH_PACKAGE} HOROVOD_WITH_MXNET=${MXNET_PACKAGE} pip install --no-cache-dir --use-pep517 -v /horovod[spark,ray]" && \
  ldconfig
 
 # Show the effective python package version to easily spot version differences

diff --git a/docker/horovod-cpu/Dockerfile b/docker/horovod-cpu/Dockerfile
@@ -81,8 +81,7 @@ RUN pip install --no-cache-dir ${PYSPARK_PACKAGE}
 # Install Horovod
 WORKDIR /horovod
 COPY . .
-RUN python setup.py sdist && \
- bash -c "HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} HOROVOD_WITH_MXNET=${MXNET_VERSION} pip install --no-cache-dir -v $(ls /horovod/dist/horovod-*.tar.gz)[spark,ray]" && \
+RUN bash -c "HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} HOROVOD_WITH_MXNET=${MXNET_VERSION} pip install --no-cache-dir --use-pep517 -v /horovod[spark,ray]" && \
  horovodrun --check-build
 
 # Check all frameworks are working correctly

diff --git a/docker/horovod-nvtabular/Dockerfile b/docker/horovod-nvtabular/Dockerfile
@@ -196,10 +196,8 @@ RUN if [[ ${MXNET_PACKAGE} == "mxnet-nightly-cu"* ]]; then \
  fi
 
 # Install Horovod.
-RUN cd /horovod && \
- python setup.py sdist && \
- ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
- bash -c "${HOROVOD_BUILD_FLAGS} HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} HOROVOD_WITH_MXNET=${MXNET_VERSION} pip install --no-cache-dir -v $(ls /horovod/dist/horovod-*.tar.gz)[spark,ray]" && \
+RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
+ bash -c "${HOROVOD_BUILD_FLAGS} HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} HOROVOD_WITH_MXNET=${MXNET_VERSION} pip install --no-cache-dir --use-pep517 -v /horovod[spark,ray]" && \
  ldconfig
 
 # Show the effective python package version to easily spot version differences

diff --git a/docker/horovod-ray/Dockerfile b/docker/horovod-ray/Dockerfile
@@ -54,9 +54,8 @@ WORKDIR /horovod
 
 # Install Horovod, temporarily using CUDA stubs
 COPY --chown=ray:users . .
-RUN python setup.py sdist && \
- sudo ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
- HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} pip install --no-cache-dir -v $(ls /horovod/dist/horovod-*.tar.gz)[ray] && \
+RUN sudo ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
+ HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} pip install --no-cache-dir --use-pep517 -v /horovod[ray] && \
  horovodrun --check-build && \
  sudo ldconfig
 

diff --git a/docker/horovod/Dockerfile b/docker/horovod/Dockerfile
@@ -100,9 +100,8 @@ RUN pip install --no-cache-dir ${PYSPARK_PACKAGE}
 # Install Horovod, temporarily using CUDA stubs
 WORKDIR /horovod
 COPY . .
-RUN python setup.py sdist && \
- ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
- bash -c "HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} HOROVOD_WITH_MXNET=${MXNET_VERSION} pip install --no-cache-dir -v $(ls /horovod/dist/horovod-*.tar.gz)[spark,ray]" && \
+RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \
+ bash -c "HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_TENSORFLOW=${TENSORFLOW_VERSION} HOROVOD_WITH_PYTORCH=${PYTORCH_VERSION} HOROVOD_WITH_MXNET=${MXNET_VERSION} pip install --no-cache-dir --use-pep517 -v /horovod[spark,ray]" && \
  horovodrun --check-build && \
  ldconfig