From a67fdb171d22a18d060e8e2d03828bc2d8105880 Mon Sep 17 00:00:00 2001 From: martinRenou Date: Mon, 24 Jan 2022 13:05:33 +0100 Subject: [PATCH] Hide visual regression tests behind a flag --- .github/workflows/test.yaml | 2 +- conftest.py | 54 +++++++++++++++++++++++++++++++++++- docrepr/tests/test_output.py | 47 ++----------------------------- 3 files changed, 57 insertions(+), 46 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b0a90f4..61cab88 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -49,7 +49,7 @@ jobs: run: python -m playwright install chromium - name: Run tests shell: bash - run: python -I -bb -X dev -W error -m pytest + run: python -I -bb -X dev -W error -m pytest --compare-screenshots - name: Upload UI Test artifacts if: failure() uses: actions/upload-artifact@v2 diff --git a/conftest.py b/conftest.py index 7c2594b..f6475d8 100644 --- a/conftest.py +++ b/conftest.py @@ -1,16 +1,19 @@ """Setup for Pytest.""" # Standard library imports +from pathlib import Path import warnings import webbrowser # Third party imports import pytest - +from PIL import Image, ImageChops +from playwright.sync_api import sync_playwright # ---- Constants OPEN_BROWSER_OPTION = '--open-browser' +COMPARE_SCREENSHOTS_OPTION = "--compare-screenshots" # ---- Pytest hooks @@ -19,6 +22,7 @@ def pytest_addoption(parser): """Add an option to open the user's web browser with HTML test output.""" parser.addoption( OPEN_BROWSER_OPTION, + COMPARE_SCREENSHOTS_OPTION, action='store_true', default=False, help='For tests that generate HTML output, open it in a web browser', @@ -37,3 +41,51 @@ def _open_browser(url): webbrowser.open_new_tab(url) return _open_browser + + +@pytest.fixture +def compare_screenshots(request): + """Do visual regression tests on the output.""" + def _compare_screenshots(test_id, url): + if request.config.getoption(COMPARE_SCREENSHOTS_OPTION): + # Filtering warnings generated by playwright + warnings.filterwarnings( + 'ignore', category=DeprecationWarning, module='pyee.*') + warnings.filterwarnings( + 'ignore', category=ResourceWarning, module='subprocess.*') + warnings.filterwarnings( + 'ignore', category=ResourceWarning, module='asyncio.*') + + test_dir = Path(__file__).parent / 'docrepr' / 'tests' + image = f'test-{test_id}.png' + reference = (test_dir / 'references' / image).resolve() + screenshot = (test_dir / 'screenshots' / image).resolve() + diff = (test_dir / 'diffs' / image).resolve() + + # Create diff directory + (test_dir / 'diffs').mkdir(parents=True, exist_ok=True) + + # Take a screenshot of the generated HTML + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page() + page.goto(f'file://{url}') + + # Wait for mathjax to finish rendering + page.wait_for_selector('#MathJax_Message', state='hidden') + + page.screenshot(path=screenshot) + browser.close() + + # Compare the screenshot with the reference + reference_im = Image.open(reference).convert('RGB') + screenshot_im = Image.open(screenshot).convert('RGB') + diff_im = ImageChops.difference(screenshot_im, reference_im) + + bbox = diff_im.getbbox() + if bbox is not None: + diff_im.save(diff) + + assert bbox is None, \ + f'{test_id} screenshot and reference do not match' + return _compare_screenshots diff --git a/docrepr/tests/test_output.py b/docrepr/tests/test_output.py index 3398621..6861cfe 100644 --- a/docrepr/tests/test_output.py +++ b/docrepr/tests/test_output.py @@ -6,12 +6,9 @@ import sys from time import sleep from pathlib import Path -import warnings # Third party imports import numpy as np -from PIL import Image, ImageChops -from playwright.sync_api import sync_playwright import pytest from IPython.core.oinspect import Inspector, object_info @@ -212,7 +209,7 @@ def _set_docrepr_options(**docrepr_options): ids=list(TEST_CASES.keys()), ) def test_sphinxify( - build_oinfo, set_docrepr_options, open_browser, + build_oinfo, set_docrepr_options, open_browser, compare_screenshots, test_id, obj, oinfo_data, docrepr_options, ): if (oinfo_data.get("docstring", None) == PLOT_DOCSTRING @@ -222,14 +219,6 @@ def test_sphinxify( pytest.skip( "Plot fails on Py3.6 on Windows; older version of Matplotlib?") - # Filtering warnings generated by playwright - warnings.filterwarnings( - 'ignore', category=DeprecationWarning, module='pyee.*') - warnings.filterwarnings( - 'ignore', category=ResourceWarning, module='subprocess.*') - warnings.filterwarnings( - 'ignore', category=ResourceWarning, module='asyncio.*') - oinfo = build_oinfo(obj, **oinfo_data) set_docrepr_options(**docrepr_options) @@ -242,35 +231,5 @@ def test_sphinxify( file_text = output_file.read_text(encoding='utf-8', errors='strict') assert len(file_text) > 512 - test_dir = Path(__file__).parent - image = f'test-{test_id}.png' - reference = (test_dir / 'references' / image).resolve() - screenshot = (test_dir / 'screenshots' / image).resolve() - diff = (test_dir / 'diffs' / image).resolve() - - # Create diff directory - (test_dir / 'diffs').mkdir(parents=True, exist_ok=True) - - # Take a screenshot of the generated HTML - with sync_playwright() as p: - browser = p.chromium.launch() - page = browser.new_page() - page.goto(f'file://{url}') - - # Wait for mathjax to finish rendering - page.wait_for_selector('#MathJax_Message', state='hidden') - - page.screenshot(path=screenshot) - browser.close() - - # Compare the screenshot with the reference - reference_im = Image.open(reference).convert('RGB') - screenshot_im = Image.open(screenshot).convert('RGB') - diff_im = ImageChops.difference(screenshot_im, reference_im) - - bbox = diff_im.getbbox() - if bbox is not None: - diff_im.save(diff) - - assert bbox is None, \ - f'{test_id} screenshot and reference do not match' + compare_screenshots(test_id, url) + open_browser(url)