merging

.github/actions/run-core-tests/group_2/action.yml

@@ -20,3 +20,5 @@ runs:
  modin/pandas/test/dataframe/test_pickle.py
  echo "::endgroup::"
  shell: bash -l {0}
+ - run: MODIN_RANGE_PARTITIONING=1 python -m pytest modin/pandas/test/dataframe/test_join_sort.py -k "merge"
+ shell: bash -l {0}

.github/actions/run-core-tests/group_3/action.yml

@@ -19,6 +19,6 @@ runs:
  shell: bash -l {0}
  - run: |
  echo "::group::Running experimental groupby tests (group 3)..."
- MODIN_EXPERIMENTAL_GROUPBY=1 ${{ inputs.runner }} ${{ inputs.parallel }} modin/pandas/test/test_groupby.py
+ MODIN_RANGE_PARTITIONING_GROUPBY=1 ${{ inputs.runner }} ${{ inputs.parallel }} modin/pandas/test/test_groupby.py
  echo "::endgroup::"
  shell: bash -l {0}

.github/workflows/ci-notebooks.yml

@@ -8,6 +8,7 @@ on:
  - setup.cfg
  - setup.py
  - requirements/env_hdk.yml
+ - requirements/env_unidist_linux.yml
 concurrency:
  # Cancel other jobs in the same branch. We don't care whether CI passes
  # on old commits.
@@ -28,12 +29,17 @@ jobs:
  steps:
  - uses: actions/checkout@v3
  - uses: ./.github/actions/python-only
- if: matrix.execution != 'hdk_on_native'
+ if: matrix.execution != 'hdk_on_native' && matrix.execution != 'pandas_on_unidist'
  - uses: ./.github/actions/mamba-env
  with:
  environment-file: requirements/env_hdk.yml
  activate-environment: modin_on_hdk
  if: matrix.execution == 'hdk_on_native'
+ - uses: ./.github/actions/mamba-env
+ with:
+ environment-file: requirements/env_unidist_linux.yml
+ activate-environment: modin_on_unidist
+ if: matrix.execution == 'pandas_on_unidist'
  - name: Cache datasets
  uses: actions/cache@v2
  with:
@@ -43,29 +49,29 @@ jobs:
  # replace modin with . in the tutorial requirements file for `pandas_on_ray` and
  # `pandas_on_dask` since we need Modin built from sources
  - run: sed -i 's/modin/./g' examples/tutorial/jupyter/execution/${{ matrix.execution }}/requirements.txt
- if: matrix.execution != 'hdk_on_native'
+ if: matrix.execution != 'hdk_on_native' && matrix.execution != 'pandas_on_unidist'
  # install dependencies required for notebooks execution for `pandas_on_ray` and `pandas_on_dask`
  # Override modin-spreadsheet install for now
  - run: |
  pip install -r examples/tutorial/jupyter/execution/${{ matrix.execution }}/requirements.txt
  pip install git+https://github.com/modin-project/modin-spreadsheet.git@49ffd89f683f54c311867d602c55443fb11bf2a5
- if: matrix.execution != 'hdk_on_native'
- # Build Modin from sources for `hdk_on_native`
+ if: matrix.execution != 'hdk_on_native' && matrix.execution != 'pandas_on_unidist'
+ # Build Modin from sources for `hdk_on_native` and `pandas_on_unidist`
  - run: pip install -e .
- if: matrix.execution == 'hdk_on_native'
+ if: matrix.execution == 'hdk_on_native' || matrix.execution == 'pandas_on_unidist'
  # install test dependencies
  # NOTE: If you are changing the set of packages installed here, make sure that
  # the dev requirements match them.
  - run: pip install pytest pytest-cov black flake8 flake8-print flake8-no-implicit-concat
- if: matrix.execution != 'hdk_on_native'
+ if: matrix.execution != 'hdk_on_native' && matrix.execution != 'pandas_on_unidist'
  - run: pip install flake8-print jupyter nbformat nbconvert
- if: matrix.execution == 'hdk_on_native'
+ if: matrix.execution == 'hdk_on_native' || matrix.execution == 'pandas_on_unidist'
  - run: pip list
- if: matrix.execution != 'hdk_on_native'
+ if: matrix.execution != 'hdk_on_native' && matrix.execution != 'pandas_on_unidist'
  - run: |
  conda info
  conda list
- if: matrix.execution == 'hdk_on_native'
+ if: matrix.execution == 'hdk_on_native' || matrix.execution == 'pandas_on_unidist'
  # setup kernel configuration for `pandas_on_unidist` execution with mpi backend
  - run: python examples/tutorial/jupyter/execution/${{ matrix.execution }}/setup_kernel.py
  if: matrix.execution == 'pandas_on_unidist'

.github/workflows/ci-required.yml

@@ -66,8 +66,6 @@ jobs:
  asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \
  asv_bench/benchmarks/scalability/__init__.py \
  modin/core/io \
- modin/experimental/core/execution/ray/implementations/pandas_on_ray \
- modin/experimental/core/execution/ray/implementations/pyarrow_on_ray \
  modin/pandas/series.py \
  modin/core/execution/python \
  modin/pandas/dataframe.py \
@@ -91,7 +89,6 @@ jobs:
  python scripts/doc_checker.py modin/experimental/pandas/io.py \
  modin/experimental/pandas/__init__.py
  - run: python scripts/doc_checker.py modin/core/storage_formats/base
- - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/pyarrow
  - run: python scripts/doc_checker.py modin/core/storage_formats/pandas
  - run: |
  python scripts/doc_checker.py \
@@ -108,3 +105,15 @@ jobs:
  - run: python scripts/doc_checker.py modin/experimental/core/execution/native/implementations/hdk_on_native/interchange/dataframe_protocol
  - run: python scripts/doc_checker.py modin/experimental/batch/pipeline.py
  - run: python scripts/doc_checker.py modin/logging
+
+ lint-black-isort:
+ name: lint (black and isort)
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: ./.github/actions/python-only
+ - run: pip install black>=24.1.0 isort>=5.12
+ # NOTE: keep the black command here in sync with the pre-commit hook in
+ # /contributing/pre-commit
+ - run: black --check --diff modin/ asv_bench/benchmarks scripts/doc_checker.py
+ - run: isort . --check-only

.github/workflows/ci.yml

@@ -26,17 +26,6 @@ env:
  MODIN_GITHUB_CI: true
 
 jobs:
- lint-black:
- name: lint (black)
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - uses: ./.github/actions/python-only
- - run: pip install black
- # NOTE: keep the black command here in sync with the pre-commit hook in
- # /contributing/pre-commit
- - run: black --check --diff modin/ asv_bench/benchmarks scripts/doc_checker.py
-
  lint-mypy:
  name: lint (mypy)
  runs-on: ubuntu-latest
@@ -77,7 +66,7 @@ jobs:
  - uses: ./.github/actions/upload-coverage
 
  test-clean-install:
- needs: [lint-flake8, lint-black]
+ needs: [lint-flake8]
  strategy:
  matrix:
  os:
@@ -92,14 +81,20 @@ jobs:
  - uses: actions/checkout@v3
  - uses: ./.github/actions/python-only
  - run: python -m pip install -e ".[all]"
- - name: Ensure all engines start up
+ - name: Ensure Ray and Dask engines start up
  run: |
  MODIN_ENGINE=dask python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
  MODIN_ENGINE=ray python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
+ - name: Ensure MPI engine start up
+ # Install a working MPI implementation beforehand so mpi4py can link to it
+ run: |
+ sudo apt install libmpich-dev
+ python -m pip install -e ".[mpi]"
  MODIN_ENGINE=unidist UNIDIST_BACKEND=mpi mpiexec -n 1 python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
+ if: matrix.os == 'ubuntu'
 
  test-internals:
- needs: [lint-flake8, lint-black]
+ needs: [lint-flake8]
  runs-on: ubuntu-latest
  defaults:
  run:
@@ -124,7 +119,7 @@ jobs:
  - uses: ./.github/actions/upload-coverage
 
  test-defaults:
- needs: [lint-flake8, lint-black]
+ needs: [lint-flake8]
  runs-on: ubuntu-latest
  defaults:
  run:
@@ -155,7 +150,7 @@ jobs:
  - uses: ./.github/actions/upload-coverage
 
  test-hdk:
- needs: [lint-flake8, lint-black]
+ needs: [lint-flake8]
  runs-on: ubuntu-latest
  defaults:
  run:
@@ -193,6 +188,7 @@ jobs:
  - run: python -m pytest modin/pandas/test/dataframe/test_binary.py
  - run: python -m pytest modin/pandas/test/dataframe/test_reduce.py
  - run: python -m pytest modin/pandas/test/dataframe/test_join_sort.py
+ - run: MODIN_RANGE_PARTITIONING=1 python -m pytest modin/pandas/test/dataframe/test_join_sort.py -k "merge"
  - run: python -m pytest modin/pandas/test/test_general.py
  - run: python -m pytest modin/pandas/test/dataframe/test_indexing.py
  - run: python -m pytest modin/pandas/test/test_series.py
@@ -212,7 +208,7 @@ jobs:
 
  test-asv-benchmarks:
  if: github.event_name == 'pull_request'
- needs: [lint-flake8, lint-black]
+ needs: [lint-flake8]
  runs-on: ubuntu-latest
  defaults:
  run:
@@ -256,11 +252,6 @@ jobs:
  MODIN_ASV_USE_IMPL=pandas asv run --quick --strict --show-stderr --launch-method=spawn \
  -b ^benchmarks -b ^io | tee benchmarks.log
 
- # HDK: ERR_OUT_OF_CPU_MEM: Not enough host memory to execute the query (MODIN#4270)
- # just disable test for testing - it works well in a machine with more memory
- sed -i 's/def time_groupby_agg_nunique(self, \*args, \*\*kwargs):/# def time_groupby_agg_nunique(self, *args, **kwargs):/g' benchmarks/hdk/benchmarks.py
- sed -i 's/execute(self.df.groupby(by=self.groupby_columns).agg("nunique"))/# execute(self.df.groupby(by=self.groupby_columns).agg("nunique"))/g' benchmarks/hdk/benchmarks.py
-
  # Otherwise, ASV considers that the environment has already been created, although ASV command is run for another config,
  # which requires the creation of a completely new environment. This step will be required after removing the manual environment setup step.
  rm -f -R .asv/env/
@@ -322,7 +313,7 @@ jobs:
  "${{ steps.filter.outputs.ray }}" "${{ steps.filter.outputs.dask }}" >> $GITHUB_OUTPUT
 
  test-all-unidist:
- needs: [lint-flake8, lint-black, execution-filter]
+ needs: [lint-flake8, execution-filter]
  if: github.event_name == 'push' || needs.execution-filter.outputs.unidist == 'true'
  runs-on: ubuntu-latest
  defaults:
@@ -353,7 +344,7 @@ jobs:
  - uses: actions/checkout@v3
  - uses: ./.github/actions/mamba-env
  with:
- environment-file: requirements/env_unidist.yml
+ environment-file: requirements/env_unidist_linux.yml
  activate-environment: modin_on_unidist
  python-version: ${{matrix.python-version}}
  - name: Install HDF5
@@ -376,8 +367,18 @@ jobs:
  - run: ./.github/workflows/sql_server/set_up_sql_server.sh
  # need an extra argument "genv" to set environment variables for mpiexec. We need
  # these variables to test writing to the mock s3 filesystem.
- - run: mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest modin/pandas/test/test_io.py --verbose
- - run: mpiexec -n 1 python -m pytest modin/experimental/pandas/test/test_io_exp.py
+ - uses: nick-fields/retry@v2
+ # to avoid issues with non-stable `to_csv` tests for unidist on MPI backend.
+ # for details see: https://github.com/modin-project/modin/pull/6776
+ with:
+ timeout_minutes: 15
+ max_attempts: 3
+ command: |
+ conda run --no-capture-output -n modin_on_unidist mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key \
+ -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest modin/pandas/test/test_io.py --verbose
+ - run: |
+ mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret \
+ python -m pytest modin/experimental/pandas/test/test_io_exp.py
  - run: mpiexec -n 1 python -m pytest modin/experimental/sql/test/test_sql.py
  - run: mpiexec -n 1 python -m pytest modin/test/interchange/dataframe_protocol/test_general.py
  - run: mpiexec -n 1 python -m pytest modin/test/interchange/dataframe_protocol/pandas/test_protocol.py
@@ -387,7 +388,7 @@ jobs:
  - uses: ./.github/actions/upload-coverage
 
  test-all:
- needs: [lint-flake8, lint-black, execution-filter]
+ needs: [lint-flake8, execution-filter]
  strategy:
  matrix:
  os:
@@ -521,7 +522,7 @@ jobs:
  if: matrix.os == 'windows'
 
  test-sanity:
- needs: [lint-flake8, lint-black, execution-filter]
+ needs: [lint-flake8, execution-filter]
  if: github.event_name == 'pull_request'
  strategy:
  matrix:
@@ -560,7 +561,7 @@ jobs:
  - uses: actions/checkout@v3
  - uses: ./.github/actions/mamba-env
  with:
- environment-file: ${{ matrix.execution.name == 'unidist' && 'requirements/env_unidist.yml' || 'environment-dev.yml' }}
+ environment-file: ${{ matrix.os == 'ubuntu' && matrix.execution.name == 'unidist' && 'requirements/env_unidist_linux.yml' || matrix.os == 'windows' && matrix.execution.name == 'unidist' && 'requirements/env_unidist_win.yml' || 'environment-dev.yml' }}
  activate-environment: ${{ matrix.execution.name == 'unidist' && 'modin_on_unidist' || 'modin' }}
  python-version: ${{matrix.python-version}}
  - name: Install HDF5
@@ -584,6 +585,7 @@ jobs:
  - run: MODIN_BENCHMARK_MODE=True ${{ matrix.execution.shell-ex }} modin/pandas/test/internals/test_benchmark_mode.py
  - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/pandas/test/internals/test_repartition.py
  - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/test/test_partition_api.py
+ - run: ${{ matrix.execution.shell-ex }} modin/pandas/api/extensions/test
  - name: xgboost tests
  run: |
  # TODO(https://github.com/modin-project/modin/issues/5194): Uncap xgboost
@@ -630,6 +632,15 @@ jobs:
  if: matrix.os != 'windows'
  - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/numpy/test
  - run: ${{ matrix.execution.shell-ex }} -m "not exclude_in_sanity" modin/pandas/test/test_io.py --verbose
+ if: matrix.execution.name != 'unidist'
+ - uses: nick-fields/retry@v2
+ # to avoid issues with non-stable `to_csv` tests for unidist on MPI backend.
+ # for details see: https://github.com/modin-project/modin/pull/6776
+ with:
+ timeout_minutes: 15
+ max_attempts: 3
+ command: conda run --no-capture-output -n modin_on_unidist ${{ matrix.execution.shell-ex }} -m "not exclude_in_sanity" modin/pandas/test/test_io.py --verbose
+ if: matrix.execution.name == 'unidist'
  - run: ${{ matrix.execution.shell-ex }} modin/experimental/pandas/test/test_io_exp.py
  - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/test/interchange/dataframe_protocol/test_general.py
  - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/test/interchange/dataframe_protocol/pandas/test_protocol.py
@@ -644,7 +655,7 @@ jobs:
  - uses: ./.github/actions/upload-coverage
 
  test-experimental:
- needs: [lint-flake8, lint-black]
+ needs: [lint-flake8]
  runs-on: ubuntu-latest
  defaults:
  run:
@@ -672,38 +683,8 @@ jobs:
  - run: python -m pytest modin/pandas/test/test_io.py --verbose
  - uses: ./.github/actions/upload-coverage
 
- test-pyarrow:
- needs: [lint-flake8, lint-black]
- runs-on: ubuntu-latest
- defaults:
- run:
- shell: bash -l {0}
- strategy:
- matrix:
- python-version: ["3.9"]
- env:
- MODIN_STORAGE_FORMAT: pyarrow
- MODIN_EXPERIMENTAL: "True"
- name: test (pyarrow, python ${{matrix.python-version}})
- services:
- moto:
- image: motoserver/moto
- ports:
- - 5000:5000
- env:
- AWS_ACCESS_KEY_ID: foobar_key
- AWS_SECRET_ACCESS_KEY: foobar_secret
- steps:
- - uses: actions/checkout@v3
- - uses: ./.github/actions/mamba-env
- with:
- environment-file: environment-dev.yml
- python-version: ${{matrix.python-version}}
- - run: sudo apt update && sudo apt install -y libhdf5-dev
- - run: python -m pytest modin/pandas/test/test_io.py::TestCsv --verbose
-
  test-spreadsheet:
- needs: [lint-flake8, lint-black]
+ needs: [lint-flake8]
  runs-on: ubuntu-latest
  defaults:
  run: