Add pip-compile

.github/workflows/ci-tests.yml

@@ -46,28 +46,23 @@ jobs:
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip  # ubuntu location
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles(format('requirements/ci-lint/requirements-{0}.txt', matrix.python-version)) }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Install dependencies
         run: |
-          python -m pip install nox pre_commit \
-            mypy==0.982 \
-            types-click \
-            types-pyyaml \
-            types-pkg_resources \
-            types-requests \
-            types-pytz
+          python -m pip install -r "requirements/ci-lint/requirements-${{ matrix.python-version }}.txt"
       - name: Pip info
         run: python -m pip list
 
       - name: Check requirements
         run: >
           nox
-          -db virtualenv -r
+          -db virtualenv
+          -r
           --non-interactive
           --python ${{ matrix.python-version }}
-          --session requirements-${{ matrix.python-version }}
+          --session "requirements-${{ matrix.python-version }}"
 
       - name: Sort Imports
         run: pre-commit run isort --all-files
@@ -129,14 +124,14 @@ jobs:
           CACHE_NUMBER: 2
         with:
           path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('requirements/conda/environment-{0}.yml', matrix.python-version)) }}-${{ hashFiles(format('requirements/ci-tests/requirements-{0}-pd{1}.txt', matrix.python-version, matrix.pandas-version)) }}
         id: cache
 
       - name: Cache pip
         uses: actions/cache@v3
         with:
           path: ${{ matrix.pip-cache }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles(format('requirements/ci-tests/requirements-{0}-pd{1}.txt', matrix.python-version, matrix.pandas-version)) }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
@@ -147,41 +142,23 @@ jobs:
           miniforge-version: latest
           miniforge-variant: Mambaforge
           use-mamba: true
-          activate-environment: pandera-dev
+          activate-environment: pandera-${{ matrix.python-version }}-${{ matrix.pandas-version }}
           channels: conda-forge
           channel-priority: true
           use-only-tar-bz2: true
 
-      # ray currently cannot be installed on python 3.11
-      - name: Remove Ray from Deps
-        if: ${{ matrix.python-version == '3.11' && matrix.os != 'macos-latest' }}
-        run: sed -i '/ray/d' environment.yml
-
-      - name: Remove Ray from Deps
-        if: ${{ matrix.python-version == '3.11' && matrix.os == 'macos-latest' }}
-        run: sed -i .bak '/ray/d' environment.yml
-
-      # need to install pandas via pip: conda installation is on the fritz
-      - name: Install Conda Deps [pandas 2]
-        if: ${{ matrix.pandas-version == '2.0.1' }}
+      - name: Install Deps via pip in conda
         run: |
-          mamba install -c conda-forge asv pandas geopandas bokeh
-          mamba env update -n pandera-dev -f environment.yml
-          pip install pandas==${{ matrix.pandas-version }}
-          pip install --user dask>=2023.3.2
+          mamba env update -n pandera-${{ matrix.python-version }}-${{ matrix.pandas-version }} -f "requirements/conda/environment-${{ matrix.python-version }}-pd${{ matrix.pandas-version }}.yml"
 
-      - name: Install Conda Deps
-        if: ${{ matrix.pandas-version != '2.0.1' }}
+      - name: Conda + Pip Install Info
         run: |
-          mamba install -c conda-forge asv pandas==${{ matrix.pandas-version }} geopandas bokeh
-          mamba env update -n pandera-dev -f environment.yml
-
-      - run: |
           conda info
           conda list
           conda config --show-sources
           conda config --show
           printenv | sort
+          pip list
 
       - name: Unit Tests - Core
         run: pytest tests/core ${{ env.PYTEST_FLAGS }}

environment.yml

@@ -1,91 +1,9 @@
-name: pandera-dev
+name: pandera-3.8-pd1.3.0
 channels:
   - conda-forge
-
 dependencies:
-  # environment management
+  - python==3.8
+  - anaconda
   - pip
-
-  # pandera dependencies
-  - packaging >= 20.0
-  - hypothesis >= 5.41.1
-  - numpy >= 1.19.0
-  - pandas >= 1.2.0
-  - scipy
-  - wrapt
-  - pyyaml >=5.1
-  - typing_inspect >= 0.6.0
-  - typing_extensions >= 3.7.4.3
-  - frictionless <= 4.40.8  # v5.* introduces breaking changes
-  - pyarrow
-  - pydantic < 2.0.0
-  - multimethod
-
-  # mypy extra
-  - pandas-stubs <= 1.5.2.221213
-
-  # pyspark extra
-  - pyspark >= 3.2.0
-
-  # modin extra
-  - modin
-  - protobuf
-
-  # dask extra
-  - dask
-  - distributed
-
-  # geopandas extra
-  - geopandas
-  - shapely
-
-  # fastapi extra
-  - fastapi
-
-  # testing and dependencies
-  - black >= 22.1.0
-
-  # testing
-  - isort >= 5.7.0
-  - mypy <= 0.982
-  - pylint <= 2.17.3
-  - pytest
-  - pytest-cov
-  - pytest-xdist
-  - pytest-asyncio
-  - pytz
-  - xdoctest
-  - nox
-  - importlib_metadata # required if python < 3.8
-
-  # fastapi testing
-  - uvicorn
-  - python-multipart
-
-  # documentation
-  - sphinx
-  - sphinx-panels
-  - sphinx-autodoc-typehints <= 1.14.1
-  - sphinx-copybutton
-  - recommonmark
-
-  # packaging
-  - twine
-
-  # performance testing
-  - asv >= 0.5.1
-
-  # optional
-  - pre_commit
-
   - pip:
-      - furo
-      - ray
-      - typeguard >= 3.0.2
-      - types-click
-      - types-pyyaml
-      - types-pkg_resources
-      - types-requests
-      - types-pytz
-      - jupyterlite
-      - jupyterlite_sphinx
+      - -r file:requirements/ci-tests/requirements-3.8-pd1.3.0.txt

requirements-ci-lint.in

@@ -0,0 +1,14 @@
+# dev-pre-commit
+pre_commit
+
+# dev-typecheck
+mypy <= 0.982
+pandas-stubs <= 1.5.2.221213 # sync with mypy
+types-click
+types-pyyaml
+types-pkg_resources
+types-requests
+types-pytz
+
+# dev-nox
+nox

requirements-dev.txt

@@ -1,58 +1 @@
-# This file is auto-generated from environment.yml, do not modify.
-# See that file for comments about the need/usage of each dependency.
-
-pip
-packaging >= 20.0
-hypothesis >= 5.41.1
-numpy >= 1.19.0
-pandas >= 1.2.0
-scipy
-wrapt
-pyyaml >=5.1
-typing_inspect >= 0.6.0
-typing_extensions >= 3.7.4.3
-frictionless <= 4.40.8
-pyarrow
-pydantic < 2.0.0
-multimethod
-pandas-stubs <= 1.5.2.221213
-pyspark >= 3.2.0
-modin
-protobuf
-dask
-distributed
-geopandas
-shapely
-fastapi
-black >= 22.1.0
-isort >= 5.7.0
-mypy <= 0.982
-pylint <= 2.17.3
-pytest
-pytest-cov
-pytest-xdist
-pytest-asyncio
-pytz
-xdoctest
-nox
-importlib_metadata
-uvicorn
-python-multipart
-sphinx
-sphinx-panels
-sphinx-autodoc-typehints <= 1.14.1
-sphinx-copybutton
-recommonmark
-twine
-asv >= 0.5.1
-pre_commit
-furo
-ray
-typeguard >= 3.0.2
-types-click
-types-pyyaml
-types-pkg_resources
-types-requests
-types-pytz
-jupyterlite
-jupyterlite_sphinx
+-r requirements/ci-tests/requirements-3.8.10-pd1.3.0.txt

requirements.in

@@ -0,0 +1,111 @@
+# core dependencies
+multimethod
+numpy >= 1.19.0
+packaging >= 20.0
+pandas >= 1.2.0
+pydantic < 2.0.0
+typeguard >= 3.0.2
+typing_extensions >= 3.7.4.3 ; python_version<'3.8'
+typing_inspect >= 0.6.0
+wrapt
+
+# strategies
+hypothesis >= 5.41.1
+
+# hypotheses
+scipy
+
+# io
+pyyaml >= 5.1
+black >= 22.1.0 # sync with dev-check
+frictionless <= 4.40.8 # v5.* introduces breaking changes
+
+# pyspark
+pyspark >= 3.2.0
+
+# modin
+modin # sync with modin-ray, modin-dask
+ray ; python_version<'3.11' # sync with modin-ray
+dask >= 2023.3.2 # sync with dask
+protobuf # sync with modin-ray, modin-dask
+
+# modin-ray
+modin # sync with modin, modin-dask
+ray ; python_version<'3.11' # sync with modin
+protobuf # sync with modin, modin-dask
+
+# modin-dask
+modin # sync with modin, modin-ray
+dask >= 2023.3.2 # sync with dask
+protobuf # sync with modin, modin-ray
+
+# dask
+dask # sync with dask
+distributed
+
+# mypy
+pandas-stubs <= 1.5.2.221213 # sync with dev-typecheck
+
+# fastapi
+fastapi
+
+# geopandas
+geopandas
+shapely
+
+# pyarrow
+pyarrow
+
+# notebook
+jupyterlite
+
+# dev
+pip-tools
+
+# dev-pkg
+twine
+
+# dev-check
+black >= 22.1.0 # sync with io
+isort >= 5.7.0
+pylint <= 2.17.3
+
+# dev-pre-commit
+pre_commit
+
+# dev-typecheck
+mypy <= 0.982
+pandas-stubs <= 1.5.2.221213 # sync with mypy
+types-click
+types-pyyaml
+types-pkg_resources
+types-requests
+types-pytz
+
+# dev-nox
+nox
+
+# dev-test
+pytest
+pytest-cov
+pytest-xdist
+pytest-asyncio
+pytz
+xdoctest
+importlib_metadata ; python_version<'3.8'
+
+# dev-fastapi-test
+uvicorn
+python-multipart
+
+# dev-perf-test
+asv >= 0.5.1
+
+# dev-doc
+sphinx
+sphinx-panels
+sphinx-autodoc-typehints <= 1.14.1
+sphinx-copybutton
+jupyterlite_sphinx
+recommonmark
+furo