retry tests

pytorch · Apr 4, 2024 · 855e6f1 · 855e6f1
1 parent 6177c80
commit 855e6f1
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 19 deletions.
diff --git a/.github/workflows/hvd-tests.yml b/.github/workflows/hvd-tests.yml
@@ -75,9 +75,12 @@ jobs:
  target_dir: /tmp
 
  - name: Run Tests
- shell: bash -l {0}
- run: |
- bash tests/run_cpu_tests.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ timeout_minutes: 25
+ shell: bash
+ command: bash tests/run_cpu_tests.sh
 
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v3

diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml
@@ -18,7 +18,7 @@ jobs:
  python-version: [3.8, 3.9, "3.10"]
  pytorch-version:
  [2.1.2, 2.0.1, 1.13.1, 1.12.1, 1.11.0, 1.10.0, 1.9.1, 1.8.1, 1.5.1]
- exclude: 
+ exclude:
  - pytorch-version: 1.5.1
  python-version: 3.9
  - pytorch-version: 1.5.1
@@ -78,7 +78,7 @@ jobs:
  pip install -r requirements-dev.txt
  python setup.py install
 
- # pytorch>=1.9.0,<1.11.0 is using "from setuptools import distutils; distutils.version.LooseVersion" anti-pattern 
+ # pytorch>=1.9.0,<1.11.0 is using "from setuptools import distutils; distutils.version.LooseVersion" anti-pattern
  # which raises the error: AttributeError: module 'distutils' has no attribute 'version' for setuptools>59
  bad_pth_version=$(python -c "import torch; print('.'.join(torch.__version__.split('.')[:2]) in ['1.9', '1.10'])")
  if [ "${bad_pth_version}" == "True" ]; then
@@ -92,9 +92,12 @@ jobs:
  target_dir: /tmp
 
  - name: Run Tests
- shell: bash -l {0}
- run: |
- bash tests/run_cpu_tests.sh "not test_time_profilers"
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ timeout_minutes: 25
+ shell: bash
+ command: bash tests/run_cpu_tests.sh "not test_time_profilers"
 
  # create-issue:
  # runs-on: ubuntu-latest

diff --git a/.github/workflows/tpu-tests.yml b/.github/workflows/tpu-tests.yml
@@ -89,13 +89,18 @@ jobs:
  target_dir: /tmp
 
  - name: Run Tests
- run: |
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${Python_ROOT_DIR}/lib
- export XRT_DEVICE_MAP="CPU:0;/job:localservice/replica:0/task:0/device:XLA_CPU:0"
- export XRT_WORKERS="localservice:0;grpc://localhost:40934"
-
- python -c "import torch_xla; print('torch xla version:', torch_xla.__version__)"
- bash tests/run_tpu_tests.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ timeout_minutes: 25
+ shell: bash
+ command: |
+ python -c "import torch_xla; print('torch xla version:', torch_xla.__version__)"
+ bash tests/run_tpu_tests.sh
+ env:
+ LD_LIBRARY_PATH: ${{ env.LD_LIBRARY_PATH }}:${{ env.Python_ROOT_DIR }}/lib
+ XRT_DEVICE_MAP: "CPU:0;/job:localservice/replica:0/task:0/device:XLA_CPU:0"
+ XRT_WORKERS: "localservice:0;grpc://localhost:40934"
 
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v3

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -40,7 +40,7 @@ jobs:
  fail-fast: false
  matrix:
  os: [ubuntu-latest]
- python-version: ["3.8", "3.9", "3.10", "3.11","3.12"]
+ python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
  pytorch-channel: [pytorch, pytorch-nightly]
  include:
  # includes a single build on windows
@@ -102,7 +102,7 @@ jobs:
 
  - name: Run Mypy
  # https://github.com/pytorch/ignite/pull/2780
- # 
+ #
  if: ${{ matrix.os == 'ubuntu-latest' && matrix.pytorch-channel == 'pytorch-nightly'}}
  run: |
  bash ./tests/run_code_style.sh mypy
@@ -120,8 +120,12 @@ jobs:
  cp -R /tmp/MNIST .
 
  - name: Run Tests
- run: |
- SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ timeout_minutes: 25
+ shell: bash
+ command: SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh
 
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v3