Fix optional fields in PySpark SQL #2141
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - dev | |
| - bugfix | |
| - "release/*" | |
| pull_request: | |
| branches: | |
| - main | |
| - dev | |
| - bugfix | |
| - "release/*" | |
| env: | |
| DEFAULT_PYTHON: 3.8 | |
| CI: "true" | |
| # Increase this value to reset cache if environment.yml has not changed | |
| CACHE_VERSION: 6 | |
| concurrency: | |
| group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| name: Linters (${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip # ubuntu location | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install nox pre_commit \ | |
| mypy==0.982 \ | |
| types-click \ | |
| types-pyyaml \ | |
| types-pkg_resources \ | |
| types-requests \ | |
| types-pytz | |
| - name: Pip info | |
| run: python -m pip list | |
| - name: Check requirements | |
| run: > | |
| nox | |
| -db virtualenv -r | |
| --non-interactive | |
| --python ${{ matrix.python-version }} | |
| --session requirements-${{ matrix.python-version }} | |
| - name: Sort Imports | |
| run: pre-commit run isort --all-files | |
| - name: Black | |
| if: always() | |
| run: pre-commit run black --all-files | |
| - name: Pylint | |
| if: always() | |
| run: pre-commit run pylint --all-files | |
| - name: Mypy Type Checking | |
| if: always() | |
| run: pre-commit run mypy --all-files | |
| tests: | |
| name: > | |
| CI Tests (${{ matrix.python-version }}, ${{ matrix.os }}, pandas-${{ matrix.pandas-version }}, pydantic-${{ matrix.pydantic-version }}) | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: pwsh | |
| env: | |
| PYTHONUTF8: 1 | |
| NOX_FLAGS: -v --no-install --no-venv --non-interactive | |
| PYTEST_FLAGS: --cov=pandera --cov-report=term-missing --cov-report=xml --cov-append | |
| HYPOTHESIS_FLAGS: -n=auto -q --hypothesis-profile=ci | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| pandas-version: ["1.5.3", "2.0.3"] | |
| pydantic-version: ["1.10.11", "2.3.0"] | |
| include: | |
| - os: ubuntu-latest | |
| pip-cache: ~/.cache/pip | |
| - os: macos-latest | |
| pip-cache: ~/Library/Caches/pip | |
| - os: windows-latest | |
| pip-cache: ~/AppData/Local/pip/Cache | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache conda | |
| uses: actions/cache@v3 | |
| env: | |
| # Increase this value to reset cache if etc/environment.yml has not changed | |
| CACHE_NUMBER: 2 | |
| with: | |
| path: ~/conda_pkgs_dir | |
| key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} | |
| id: cache | |
| - name: Cache pip | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ matrix.pip-cache }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install deps | |
| run: pip install -r ci/requirements-py${{ matrix.python-version }}-pandas${{ matrix.pandas-version }}-pydantic${{ matrix.pydantic-version }}.txt | |
| - run: | | |
| pip list | |
| printenv | sort | |
| - name: Unit Tests - Core | |
| run: pytest tests/core ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Hypotheses | |
| run: pytest tests/hypotheses ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - IO | |
| run: pytest tests/io ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Mypy | |
| # mypy tests hang on windows | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| run: pytest -v tests/mypy ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Strategies | |
| run: pytest tests/strategies ${{ env.PYTEST_FLAGS }} ${{ env.HYPOTHESIS_FLAGS }} | |
| - name: Unit Tests - FastAPI | |
| # there's an issue with the fastapi tests in CI that's not reproducible locally | |
| # when pydantic > v2 | |
| run: pytest tests/fastapi ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - GeoPandas | |
| run: pytest tests/geopandas ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Dask | |
| if: ${{ matrix.pandas-version != '2.0.3' }} | |
| run: pytest tests/dask ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Pyspark | |
| if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }} | |
| run: pytest tests/pyspark ${{ env.PYTEST_FLAGS }} | |
| - name: Unit Tests - Modin-Dask | |
| if: ${{ matrix.pandas-version != '2.0.3' }} | |
| run: pytest tests/modin ${{ env.PYTEST_FLAGS }} | |
| env: | |
| CI_MODIN_ENGINES: dask | |
| - name: Unit Tests - Modin-Ray | |
| # ray CI issues with the following: | |
| # - windows, python 3.10 | |
| # Tracking issue: https://github.com/modin-project/modin/issues/5466 | |
| if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }} | |
| run: pytest tests/modin ${{ env.PYTEST_FLAGS }} | |
| env: | |
| CI_MODIN_ENGINES: ray | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| - name: Check Docstrings | |
| if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pandas-version == '2.0.3' }} | |
| run: nox ${{ env.NOX_FLAGS }} --session doctests | |
| - name: Check Docs | |
| if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pydantic-version == '2.0.3' }} | |
| run: nox ${{ env.NOX_FLAGS }} --session docs |