Merge pull request #1059 from rapidsai/branch-22.12

.flake8

@@ -0,0 +1,14 @@
+[flake8]
+exclude = docs, __init__.py
+max-line-length = 88
+ignore =
+    # Assigning lambda expression
+    E731
+    # Ambiguous variable names
+    E741
+    # line break before binary operator
+    W503
+    # whitespace before :
+    E203
+    # whitespace after ,
+    E231

.pre-commit-config.yaml

@@ -1,13 +1,13 @@
 repos:
       - repo: https://github.com/pycqa/isort
-        rev: 5.6.4
+        rev: 5.10.1
         hooks:
               - id: isort
       - repo: https://github.com/ambv/black
         rev: 22.3.0
         hooks:
               - id: black
-      - repo: https://gitlab.com/pycqa/flake8
+      - repo: https://github.com/PyCQA/flake8
         rev: 3.8.3
         hooks:
               - id: flake8

CHANGELOG.md

@@ -1,3 +1,44 @@
+# dask-cuda 22.12.00 (8 Dec 2022)
+
+## 🚨 Breaking Changes
+
+- Make local_directory a required argument for spilling impls ([#1023](https://github.com/rapidsai/dask-cuda/pull/1023)) [@wence-](https://github.com/wence-)
+
+## 🐛 Bug Fixes
+
+- Fix `parse_memory_limit` function call ([#1055](https://github.com/rapidsai/dask-cuda/pull/1055)) [@galipremsagar](https://github.com/galipremsagar)
+- Work around Jupyter errors in CI ([#1041](https://github.com/rapidsai/dask-cuda/pull/1041)) [@pentschev](https://github.com/pentschev)
+- Fix version constraint ([#1036](https://github.com/rapidsai/dask-cuda/pull/1036)) [@wence-](https://github.com/wence-)
+- Support the new `Buffer` in cudf ([#1033](https://github.com/rapidsai/dask-cuda/pull/1033)) [@madsbk](https://github.com/madsbk)
+- Install Dask nightly last in CI ([#1029](https://github.com/rapidsai/dask-cuda/pull/1029)) [@pentschev](https://github.com/pentschev)
+- Fix recorded time in merge benchmark ([#1028](https://github.com/rapidsai/dask-cuda/pull/1028)) [@wence-](https://github.com/wence-)
+- Switch pre-import not found test to sync definition ([#1026](https://github.com/rapidsai/dask-cuda/pull/1026)) [@pentschev](https://github.com/pentschev)
+- Make local_directory a required argument for spilling impls ([#1023](https://github.com/rapidsai/dask-cuda/pull/1023)) [@wence-](https://github.com/wence-)
+- Fixes for handling MIG devices ([#950](https://github.com/rapidsai/dask-cuda/pull/950)) [@pentschev](https://github.com/pentschev)
+
+## 📖 Documentation
+
+- Merge 22.10 into 22.12 ([#1016](https://github.com/rapidsai/dask-cuda/pull/1016)) [@pentschev](https://github.com/pentschev)
+- Merge 22.08 into 22.10 ([#1010](https://github.com/rapidsai/dask-cuda/pull/1010)) [@pentschev](https://github.com/pentschev)
+
+## 🚀 New Features
+
+- Allow specifying fractions as RMM pool initial/maximum size ([#1021](https://github.com/rapidsai/dask-cuda/pull/1021)) [@pentschev](https://github.com/pentschev)
+- Add feature to get cluster configuration ([#1006](https://github.com/rapidsai/dask-cuda/pull/1006)) [@quasiben](https://github.com/quasiben)
+- Add benchmark option to use dask-noop ([#994](https://github.com/rapidsai/dask-cuda/pull/994)) [@wence-](https://github.com/wence-)
+
+## 🛠️ Improvements
+
+- Ensure linting checks for whole repo in CI ([#1053](https://github.com/rapidsai/dask-cuda/pull/1053)) [@pentschev](https://github.com/pentschev)
+- Pin `dask` and `distributed` for release ([#1046](https://github.com/rapidsai/dask-cuda/pull/1046)) [@galipremsagar](https://github.com/galipremsagar)
+- Remove `pytest-asyncio` dependency ([#1045](https://github.com/rapidsai/dask-cuda/pull/1045)) [@pentschev](https://github.com/pentschev)
+- Migrate as much as possible to `pyproject.toml` ([#1035](https://github.com/rapidsai/dask-cuda/pull/1035)) [@jakirkham](https://github.com/jakirkham)
+- Re-implement shuffle using staging ([#1030](https://github.com/rapidsai/dask-cuda/pull/1030)) [@madsbk](https://github.com/madsbk)
+- Explicit-comms-shuffle: fine control of task scheduling ([#1025](https://github.com/rapidsai/dask-cuda/pull/1025)) [@madsbk](https://github.com/madsbk)
+- Remove stale labeler ([#1024](https://github.com/rapidsai/dask-cuda/pull/1024)) [@raydouglass](https://github.com/raydouglass)
+- Unpin `dask` and `distributed` for development ([#1005](https://github.com/rapidsai/dask-cuda/pull/1005)) [@galipremsagar](https://github.com/galipremsagar)
+- Support cuDF's built-in spilling ([#984](https://github.com/rapidsai/dask-cuda/pull/984)) [@madsbk](https://github.com/madsbk)
+
 # dask-cuda 22.10.00 (12 Oct 2022)
 
 ## 🐛 Bug Fixes
@@ -23,6 +64,7 @@
 - Unpin `dask` and `distributed` for development ([#971](https://github.com/rapidsai/dask-cuda/pull/971)) [@galipremsagar](https://github.com/galipremsagar)
 
 # dask-cuda 22.08.00 (17 Aug 2022)
+
 ## 🚨 Breaking Changes
 
 - Fix useless property ([#944](https://github.com/rapidsai/dask-cuda/pull/944)) [@wence-](https://github.com/wence-)

MANIFEST.in

@@ -1,3 +1 @@
-include requirements.txt
-include versioneer.py
 include dask_cuda/_version.py

ci/checks/style.sh

@@ -12,50 +12,5 @@ PATH=/opt/conda/bin:$PATH
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
 
-# Print versions
-echo -e "\nVersions:"
-black --version
-echo "isort, `isort --vn`"
-echo "flake8, `flake8 --version`"
-
-# Run isort and get results/return code
-ISORT=`isort --check-only dask_cuda 2>&1`
-ISORT_RETVAL=$?
-
-# Run black and get results/return code
-BLACK=`black --check dask_cuda 2>&1`
-BLACK_RETVAL=$?
-
-# Run flake8 and get results/return code
-FLAKE=`flake8 dask_cuda 2>&1`
-FLAKE_RETVAL=$?
-
-# Output results if failure otherwise show pass
-if [ "$ISORT_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n"
-  echo -e "$ISORT"
-  echo -e "\n\n>>>> FAILED: isort style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort style check\n\n"
-fi
-
-if [ "$BLACK_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: black style check; begin output\n\n"
-  echo -e "$BLACK"
-  echo -e "\n\n>>>> FAILED: black style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: black style check\n\n"
-fi
-
-if [ "$FLAKE_RETVAL" != "0" ]; then
-  echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
-  echo -e "$FLAKE"
-  echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
-fi
-
-RETVALS=($ISORT_RETVAL $BLACK_RETVAL $FLAKE_RETVAL)
-IFS=$'\n'
-RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
-exit $RETVAL
+# Run pre-commit checks
+pre-commit run --hook-stage manual --all-files

ci/cpu/build.sh

@@ -23,6 +23,9 @@ export GPUCI_CONDA_RETRY_SLEEP=30
 # `dask/label/dev` channel is removed.
 export INSTALL_DASK_MAIN=0
 
+# Dask version to install when `INSTALL_DASK_MAIN=0`
+export DASK_STABLE_VERSION="2022.11.1"
+
 # Switch to project root; also root of repo checkout
 cd "$WORKSPACE"
 
@@ -69,8 +72,17 @@ conda list --show-channel-urls
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
-pip install git+https://github.com/dask/[email protected]
-pip install git+https://github.com/dask/[email protected]
+# Install latest nightly version for dask and distributed if needed
+if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
+  gpuci_logger "Installing dask and distributed from dask nightly channel"
+  gpuci_mamba_retry install -c dask/label/dev \
+    "dask/label/dev::dask" \
+    "dask/label/dev::distributed"
+else
+  gpuci_logger "gpuci_mamba_retry install conda-forge::dask==${DASK_STABLE_VERSION} conda-forge::distributed==${DASK_STABLE_VERSION} conda-forge::dask-core==${DASK_STABLE_VERSION} --force-reinstall"
+  gpuci_mamba_retry install conda-forge::dask==${DASK_STABLE_VERSION} conda-forge::distributed==${DASK_STABLE_VERSION} conda-forge::dask-core==${DASK_STABLE_VERSION} --force-reinstall
+fi
+
 
 ################################################################################
 # BUILD - Package builds

ci/gpu/build.sh

@@ -26,7 +26,7 @@ cd "$WORKSPACE"
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 export UCX_PATH=$CONDA_PREFIX
-export UCXPY_VERSION=0.28.*
+export UCXPY_VERSION=0.29.*
 unset GIT_DESCRIBE_TAG
 
 # Enable NumPy's __array_function__ protocol (needed for NumPy 1.16.x,
@@ -38,7 +38,11 @@ export NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1
 export INSTALL_DASK_MAIN=0
 
 # Dask version to install when `INSTALL_DASK_MAIN=0`
-export DASK_STABLE_VERSION="2022.9.2"
+export DASK_STABLE_VERSION="2022.11.1"
+
+# Temporary workaround for Jupyter errors.
+# See https://github.com/rapidsai/dask-cuda/issues/1040
+export JUPYTER_PLATFORM_DIRS=1
 
 ################################################################################
 # SETUP - Check environment
@@ -59,26 +63,11 @@ conda config --show-sources
 conda list --show-channel-urls
 
 # Installing cucim in order to test GDS spilling
-# Pin pytest-asyncio because latest versions modify the default asyncio
-# `event_loop_policy`. See https://github.com/dask/distributed/pull/4212 .
 gpuci_mamba_retry install "cudf=${MINOR_VERSION}" \
               "dask-cudf=${MINOR_VERSION}" \
               "ucx-py=${UCXPY_VERSION}" \
               "ucx-proc=*=gpu" \
-              "cucim" \
-              "pytest-asyncio=<0.14.0"
-
-# Install latest nightly version for dask and distributed if needed
-if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
-  gpuci_logger "Installing dask and distributed from dask nightly channel"
-  gpuci_mamba_retry install -c dask/label/dev \
-    "dask/label/dev::dask" \
-    "dask/label/dev::distributed"
-else
-  gpuci_logger "gpuci_mamba_retry install conda-forge::dask==${DASK_STABLE_VERSION} conda-forge::distributed==${DASK_STABLE_VERSION} conda-forge::dask-core==${DASK_STABLE_VERSION} --force-reinstall"
-  gpuci_mamba_retry install conda-forge::dask==${DASK_STABLE_VERSION} conda-forge::distributed==${DASK_STABLE_VERSION} conda-forge::dask-core==${DASK_STABLE_VERSION} --force-reinstall
-  conda config --system --remove channels dask/label/dev
-fi
+              "cucim"
 
 
 gpuci_logger "Check versions"
@@ -103,6 +92,21 @@ CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
 gpuci_conda_retry mambabuild --croot "${CONDA_BLD_DIR}" conda/recipes/dask-cuda --python="${PYTHON}"
 gpuci_mamba_retry install -c "${CONDA_BLD_DIR}" dask-cuda
 
+################################################################################
+# DASK - Install latest nightly version for dask and distributed if needed.
+#        Done after everything else to ensure packages are not downgraded.
+################################################################################
+if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
+  gpuci_logger "Installing dask and distributed from dask nightly channel"
+  gpuci_mamba_retry install -c dask/label/dev \
+    "dask/label/dev::dask" \
+    "dask/label/dev::distributed"
+else
+  gpuci_logger "gpuci_mamba_retry install conda-forge::dask==${DASK_STABLE_VERSION} conda-forge::distributed==${DASK_STABLE_VERSION} conda-forge::dask-core==${DASK_STABLE_VERSION} --force-reinstall"
+  gpuci_mamba_retry install conda-forge::dask==${DASK_STABLE_VERSION} conda-forge::distributed==${DASK_STABLE_VERSION} conda-forge::dask-core==${DASK_STABLE_VERSION} --force-reinstall
+  conda config --system --remove channels dask/label/dev
+fi
+
 ################################################################################
 # TEST - Run pytests for ucx-py
 ################################################################################

conda/recipes/dask-cuda/meta.yaml

@@ -2,10 +2,13 @@
 
 # Usage:
 #   conda build -c conda-forge .
-{% set data = load_setup_py_data() %}
+{% set data = load_file_data("pyproject.toml") %}
 
 {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
-{% set py_version=environ.get('CONDA_PY', 36) %}
+{% set number = environ.get('GIT_DESCRIBE_NUMBER', 0) %}
+{% set py_version = environ.get('CONDA_PY', 36) %}
+{% set git_hash = environ.get('GIT_DESCRIBE_HASH', '') %}
+
 package:
   name: dask-cuda
   version: {{ version }}
@@ -14,16 +17,24 @@ source:
   git_url: ../../..
 
 build:
-  number: {{ GIT_DESCRIBE_NUMBER }}
-  string: py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  number: {{ number }}
+  string: py{{ py_version }}_{{ git_hash }}_{{ number }}
+  script:
+    - {{ PYTHON }} -m pip install . -vv
+  entry_points:
+    {% for e in data.get("project", {}).get("scripts", {}).items() %}
+    - {{ e|join(" = ") }}
+    {% endfor %}
 
 requirements:
   host:
     - python
-    - setuptools
+    - pip
+    - tomli
+    - versioneer >=0.24
   run:
     - python
-    {% for r in data.get("install_requires", []) %}
+    {% for r in data.get("project", {}).get("dependencies", []) %}
     - {{ r }}
     {% endfor %}