[BUGFIX] [PYSPARK] Avoid running nullable checks if `nullable=True` #2178

Workflow file for this run

.github/workflows/ci-tests.yml at 38e0b1b

	name: CI Tests
	on:
	push:
	branches:
	- main
	- dev
	- bugfix
	- "release/*"
	pull_request:
	branches:
	- main
	- dev
	- bugfix
	- "release/*"

	env:
	DEFAULT_PYTHON: 3.8
	CI: "true"
	# Increase this value to reset cache if environment.yml has not changed
	CACHE_VERSION: 6

	concurrency:
	group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label \|\| github.head_ref \|\| github.ref }}'
	cancel-in-progress: true

	jobs:

	lint:
	name: Linters (${{ matrix.python-version }})
	runs-on: ubuntu-latest
	strategy:
	fail-fast: true
	matrix:
	python-version: ["3.8", "3.9", "3.10", "3.11"]
	defaults:
	run:
	shell: bash -l {0}

	steps:
	- uses: actions/checkout@v3

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- uses: actions/cache@v3
	with:
	path: ~/.cache/pip # ubuntu location
	key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install dependencies
	run: \|
	python -m pip install nox pre_commit \
	mypy==0.982 \
	types-click \
	types-pyyaml \
	types-pkg_resources \
	types-requests \
	types-pytz
	- name: Pip info
	run: python -m pip list

	- name: Check requirements
	run: >
	nox
	-db virtualenv -r
	--non-interactive
	--python ${{ matrix.python-version }}
	--session requirements-${{ matrix.python-version }}

	- name: Sort Imports
	run: pre-commit run isort --all-files

	- name: Black
	if: always()
	run: pre-commit run black --all-files

	- name: Pylint
	if: always()
	run: pre-commit run pylint --all-files

	- name: Mypy Type Checking
	if: always()
	run: pre-commit run mypy --all-files

	tests:
	name: >
	CI Tests (${{ matrix.python-version }}, ${{ matrix.os }}, pandas-${{ matrix.pandas-version }}, pydantic-${{ matrix.pydantic-version }})
	runs-on: ${{ matrix.os }}
	defaults:
	run:
	shell: pwsh
	env:
	PYTHONUTF8: 1
	NOX_FLAGS: -v --no-install --no-venv --non-interactive
	PYTEST_FLAGS: --cov=pandera --cov-report=term-missing --cov-report=xml --cov-append
	HYPOTHESIS_FLAGS: -n=auto -q --hypothesis-profile=ci
	strategy:
	fail-fast: true
	matrix:
	os: ["ubuntu-latest", "macos-latest", "windows-latest"]
	python-version: ["3.8", "3.9", "3.10", "3.11"]
	pandas-version: ["1.5.3", "2.0.3"]
	pydantic-version: ["1.10.11", "2.3.0"]
	include:
	- os: ubuntu-latest
	pip-cache: ~/.cache/pip
	- os: macos-latest
	pip-cache: ~/Library/Caches/pip
	- os: windows-latest
	pip-cache: ~/AppData/Local/pip/Cache

	steps:
	- uses: actions/checkout@v3

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Cache conda
	uses: actions/cache@v3
	env:
	# Increase this value to reset cache if etc/environment.yml has not changed
	CACHE_NUMBER: 2
	with:
	path: ~/conda_pkgs_dir
	key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
	id: cache

	- name: Cache pip
	uses: actions/cache@v3
	with:
	path: ${{ matrix.pip-cache }}
	key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-

	- name: Install deps
	run: pip install -r ci/requirements-py${{ matrix.python-version }}-pandas${{ matrix.pandas-version }}-pydantic${{ matrix.pydantic-version }}.txt

	- run: \|
	pip list
	printenv \| sort

	- name: Unit Tests - Core
	run: pytest tests/core ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - Hypotheses
	run: pytest tests/hypotheses ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - IO
	run: pytest tests/io ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - Mypy
	# mypy tests hang on windows
	if: ${{ matrix.os != 'windows-latest' }}
	run: pytest -v tests/mypy ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - Strategies
	run: pytest tests/strategies ${{ env.PYTEST_FLAGS }} ${{ env.HYPOTHESIS_FLAGS }}

	- name: Unit Tests - FastAPI
	# there's an issue with the fastapi tests in CI that's not reproducible locally
	# when pydantic > v2
	run: pytest tests/fastapi ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - GeoPandas
	run: pytest tests/geopandas ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - Dask
	if: ${{ matrix.pandas-version != '2.0.3' }}
	run: pytest tests/dask ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - Pyspark
	if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }}
	run: pytest tests/pyspark ${{ env.PYTEST_FLAGS }}

	- name: Unit Tests - Modin-Dask
	if: ${{ matrix.pandas-version != '2.0.3' }}
	run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
	env:
	CI_MODIN_ENGINES: dask

	- name: Unit Tests - Modin-Ray
	# ray CI issues with the following:
	# - windows, python 3.10
	# Tracking issue: https://github.com/modin-project/modin/issues/5466
	if: ${{ matrix.os != 'windows-latest' && matrix.pandas-version != '2.0.3' }}
	run: pytest tests/modin ${{ env.PYTEST_FLAGS }}
	env:
	CI_MODIN_ENGINES: ray

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3

	- name: Check Docstrings
	if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pandas-version == '2.0.3' }}
	run: nox ${{ env.NOX_FLAGS }} --session doctests

	- name: Check Docs
	if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pydantic-version == '2.0.3' }}
	run: nox ${{ env.NOX_FLAGS }} --session docs

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[BUGFIX] [PYSPARK] Avoid running nullable checks if `nullable=True` #2178

Workflow file

[BUGFIX] [PYSPARK] Avoid running nullable checks if `nullable=True` #2178

Jobs

Run details

Workflow file for this run

[BUGFIX] [PYSPARK] Avoid running nullable checks if nullable=True #2178

Workflow file

Workflow file for this run

[BUGFIX] [PYSPARK] Avoid running nullable checks if `nullable=True` #2178