Skip to content

Commit

Permalink
Visual regression tests with playwright
Browse files Browse the repository at this point in the history
  • Loading branch information
martinRenou committed Jan 25, 2022
1 parent 0266836 commit 23d1039
Show file tree
Hide file tree
Showing 50 changed files with 199 additions and 8 deletions.
8 changes: 8 additions & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
## Visual changes
If this PR has an impact on the outputs, you can ask the bot to update
the references for the visual regression tests commenting the following:
"Please update reference screenshots"
-->
43 changes: 43 additions & 0 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,52 @@ jobs:
- name: List packages in environment
run: pip list
- name: Run tests
if: ${{ matrix.python-version == '3.6' }}
shell: bash
run: python -I -bb -X dev -W error -m pytest
- name: Run tests
if: ${{ matrix.python-version != '3.6' }}
shell: bash
run: python -I -bb -X dev -W error -m pytest --asyncio-mode=auto
- name: Twine check
run: pipx run twine check --strict dist/*
- name: Pip check
run: pip check -v

visual-regression-tests:
name: Visual Regression Tests ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
steps:
- name: Checkout branch
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Print basic Python info
shell: python
run: |
import sys
print(sys.executable, sys.version, sep='\n')
- name: Install docrepr
run: python -m pip install --upgrade .[test,visual_tests]
- name: List packages in environment
run: pip list
- name: Install Firefox
run: python -m playwright install firefox
- name: Run tests
shell: bash
run: python -I -bb -X dev -W error -m pytest --compare-screenshots --asyncio-mode=auto
- name: Upload UI Test artifacts
if: failure()
uses: actions/upload-artifact@v2
with:
name: visual-regression-test-output
path: |
docrepr/tests/references/**
docrepr/tests/screenshots/**
docrepr/tests/diffs/**
56 changes: 56 additions & 0 deletions .github/workflows/update_references.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
name: Update references

on:
issue_comment:
types: [created, edited]


jobs:
update_references:

name: PR comment
if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'Please update reference screenshots') }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
steps:

- name: Checkout repository
uses: actions/checkout@v2
- name: Checkout the branch from the PR that triggered the job
run: hub pr checkout ${{ github.event.issue.number }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Print basic Python info
shell: python
run: |
import sys
print(sys.executable, sys.version, sep='\n')
- name: Install docrepr
run: python -m pip install --upgrade .[test,visual_tests]
- name: List packages in environment
run: pip list
- name: Install Firefox
run: python -m playwright install firefox
- name: Run tests
shell: bash
run: python -I -bb -X dev -W error -m pytest --update-reference-screenshots --asyncio-mode=auto
- name: Commit reference images
run: |
git config user.name 'github-actions[bot]'
git config user.email 'github-actions[bot]@users.noreply.github.com'
git pull
git add docrepr/tests/references
git commit -m "Update references for ${{ matrix.os }}"
git push
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ pip-delete-this-directory.txt
coverage.xml
htmlcov/
nosetests.xml
docrepr/tests/screenshots/**
docrepr/tests/diffs/**

# Translations
*.mo
Expand Down
77 changes: 75 additions & 2 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,45 @@
"""Setup for Pytest."""

# Standard library imports
from pathlib import Path
import shutil
from sys import platform
import warnings
import webbrowser

# Third party imports
import pytest

from PIL import Image, ImageChops

# ---- Constants

OPEN_BROWSER_OPTION = '--open-browser'
COMPARE_SCREENSHOTS_OPTION = "--compare-screenshots"
UPDATE_REFERENCE_SCREENSHOTS_OPTION = "--update-reference-screenshots"


# ---- Pytest hooks

def pytest_addoption(parser):
"""Add an option to open the user's web browser with HTML test output."""
"""Add command line options."""
parser.addoption(
OPEN_BROWSER_OPTION,
action='store_true',
default=False,
help='For tests that generate HTML output, open it in a web browser',
)
parser.addoption(
COMPARE_SCREENSHOTS_OPTION,
action='store_true',
default=False,
help='For tests that generate HTML output, run visual regression tests on them',
)
parser.addoption(
UPDATE_REFERENCE_SCREENSHOTS_OPTION,
action='store_true',
default=False,
help='For tests that generate HTML output, update reference screenshots for the visual regression tests',
)


# ---- Fixtures
Expand All @@ -37,3 +54,59 @@ def _open_browser(url):

webbrowser.open_new_tab(url)
return _open_browser


@pytest.fixture
async def compare_screenshots(request):
"""Run visual regression test on the output."""
async def _compare_screenshots(test_id, url):
if (request.config.getoption(COMPARE_SCREENSHOTS_OPTION) or
request.config.getoption(UPDATE_REFERENCE_SCREENSHOTS_OPTION)):
from playwright.async_api import async_playwright

# Filtering warnings generated by playwright
warnings.filterwarnings(
'ignore', category=DeprecationWarning, module='pyee.*')

test_dir = Path(__file__).parent / 'docrepr' / 'tests'
image = f'test-{test_id}-{platform}.png'
reference = (test_dir / 'references' / image).resolve()
screenshot = (test_dir / 'screenshots' / image).resolve()
diff = (test_dir / 'diffs' / image).resolve()

# Create diff directory
(test_dir / 'diffs').mkdir(parents=True, exist_ok=True)

# Take a screenshot of the generated HTML
async with async_playwright() as p:
browser = await p.firefox.launch()
page = await browser.new_page()
await page.goto(f'file://{url}', wait_until="networkidle")

# Wait for mathjax to finish rendering
await page.wait_for_selector('#MathJax_Message', state='hidden')

await page.screenshot(path=screenshot)
await browser.close()

# Compress the screenshot
screenshot_im = Image.open(screenshot).convert('RGB')
screenshot_im.save(screenshot, optimize=True, quality=70)

# If a reference update has been requested, we copy the screenshot
# and consider the test as "passing"
if request.config.getoption(UPDATE_REFERENCE_SCREENSHOTS_OPTION):
shutil.copyfile(screenshot, reference)
return

# Compare the screenshot with the reference
reference_im = Image.open(reference).convert('RGB')
diff_im = ImageChops.difference(screenshot_im, reference_im)

bbox = diff_im.getbbox()
if bbox is not None:
diff_im.save(diff)

assert bbox is None, \
f'{test_id} screenshot and reference do not match'
return _compare_screenshots
Binary file added docrepr/tests/references/test-basic-darwin.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docrepr/tests/references/test-basic-linux.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docrepr/tests/references/test-basic-win32.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docrepr/tests/references/test-collapse-darwin.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docrepr/tests/references/test-collapse-linux.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docrepr/tests/references/test-collapse-win32.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docrepr/tests/references/test-numpy_sin-linux.png
Binary file added docrepr/tests/references/test-numpy_sin-win32.png
Binary file added docrepr/tests/references/test-outline-darwin.png
Binary file added docrepr/tests/references/test-outline-linux.png
Binary file added docrepr/tests/references/test-outline-win32.png
Binary file added docrepr/tests/references/test-plot-darwin.png
Binary file added docrepr/tests/references/test-plot-linux.png
Binary file added docrepr/tests/references/test-plot-win32.png
16 changes: 11 additions & 5 deletions docrepr/tests/test_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import copy
import subprocess
import sys
from time import sleep
from pathlib import Path

# Third party imports
Expand Down Expand Up @@ -166,7 +167,10 @@ def eat_one(self):
# ---- Helper functions

def _test_cases_to_params(test_cases):
return [tuple(test_case.values()) for test_case in test_cases.values()]
return [
[test_id, *test_case.values()]
for test_id, test_case in test_cases.items()
]


# ---- Fixtures
Expand Down Expand Up @@ -199,14 +203,15 @@ def _set_docrepr_options(**docrepr_options):

# ---- Tests

@pytest.mark.asyncio
@pytest.mark.parametrize(
('obj', 'oinfo_data', 'docrepr_options'),
('test_id', 'obj', 'oinfo_data', 'docrepr_options'),
_test_cases_to_params(TEST_CASES),
ids=list(TEST_CASES.keys()),
)
def test_sphinxify(
build_oinfo, set_docrepr_options, open_browser,
obj, oinfo_data, docrepr_options,
async def test_sphinxify(
build_oinfo, set_docrepr_options, open_browser, compare_screenshots,
test_id, obj, oinfo_data, docrepr_options,
):
if (oinfo_data.get("docstring", None) == PLOT_DOCSTRING
and sys.version_info.major == 3
Expand All @@ -227,4 +232,5 @@ def test_sphinxify(
file_text = output_file.read_text(encoding='utf-8', errors='strict')
assert len(file_text) > 512

await compare_screenshots(test_id, url)
open_browser(url)
2 changes: 1 addition & 1 deletion pytest.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[pytest]
addopts = --durations=5 -v -r a --color=yes --code-highlight=yes --strict-config --strict-markers --import-mode=importlib --maxfail 5
addopts = --durations=5 -v -r a --color=yes --code-highlight=yes --strict-config --strict-markers --import-mode=importlib
empty_parameter_set_mark = fail_at_collect
filterwarnings =
error
Expand Down
3 changes: 3 additions & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,6 @@ test =
matplotlib>=2.2.4
numpy
pytest>=6.0.0
pytest-asyncio
visual_tests =
playwright

0 comments on commit 23d1039

Please sign in to comment.