Hide visual regression tests behind a flag

.github/workflows/test.yaml

@@ -49,7 +49,7 @@ jobs:
       run: python -m playwright install chromium
     - name: Run tests
       shell: bash
-      run: python -I -bb -X dev -W error -m pytest
+      run: python -I -bb -X dev -W error -m pytest --compare-screenshots
     - name: Upload UI Test artifacts
       if: failure()
       uses: actions/upload-artifact@v2

conftest.py

@@ -1,16 +1,19 @@
 """Setup for Pytest."""
 
 # Standard library imports
+from pathlib import Path
 import warnings
 import webbrowser
 
 # Third party imports
 import pytest
-
+from PIL import Image, ImageChops
+from playwright.sync_api import sync_playwright
 
 # ---- Constants
 
 OPEN_BROWSER_OPTION = '--open-browser'
+COMPARE_SCREENSHOTS_OPTION = "--compare-screenshots"
 
 
 # ---- Pytest hooks
@@ -19,6 +22,7 @@ def pytest_addoption(parser):
     """Add an option to open the user's web browser with HTML test output."""
     parser.addoption(
         OPEN_BROWSER_OPTION,
+        COMPARE_SCREENSHOTS_OPTION,
         action='store_true',
         default=False,
         help='For tests that generate HTML output, open it in a web browser',
@@ -37,3 +41,51 @@ def _open_browser(url):
 
             webbrowser.open_new_tab(url)
     return _open_browser
+
+
+@pytest.fixture
+def compare_screenshots(request):
+    """Do visual regression tests on the output."""
+    def _compare_screenshots(test_id, url):
+        if request.config.getoption(COMPARE_SCREENSHOTS_OPTION):
+            # Filtering warnings generated by playwright
+            warnings.filterwarnings(
+                'ignore', category=DeprecationWarning, module='pyee.*')
+            warnings.filterwarnings(
+                'ignore', category=ResourceWarning, module='subprocess.*')
+            warnings.filterwarnings(
+                'ignore', category=ResourceWarning, module='asyncio.*')
+
+            test_dir = Path(__file__).parent / 'docrepr' / 'tests'
+            image = f'test-{test_id}.png'
+            reference = (test_dir / 'references' / image).resolve()
+            screenshot = (test_dir / 'screenshots' / image).resolve()
+            diff = (test_dir / 'diffs' / image).resolve()
+
+            # Create diff directory
+            (test_dir / 'diffs').mkdir(parents=True, exist_ok=True)
+
+            # Take a screenshot of the generated HTML
+            with sync_playwright() as p:
+                browser = p.chromium.launch()
+                page = browser.new_page()
+                page.goto(f'file://{url}')
+
+                # Wait for mathjax to finish rendering
+                page.wait_for_selector('#MathJax_Message', state='hidden')
+
+                page.screenshot(path=screenshot)
+                browser.close()
+
+            # Compare the screenshot with the reference
+            reference_im = Image.open(reference).convert('RGB')
+            screenshot_im = Image.open(screenshot).convert('RGB')
+            diff_im = ImageChops.difference(screenshot_im, reference_im)
+
+            bbox = diff_im.getbbox()
+            if bbox is not None:
+                diff_im.save(diff)
+
+            assert bbox is None, \
+                f'{test_id} screenshot and reference do not match'
+    return _compare_screenshots

docrepr/tests/test_output.py

@@ -6,12 +6,9 @@
 import sys
 from time import sleep
 from pathlib import Path
-import warnings
 
 # Third party imports
 import numpy as np
-from PIL import Image, ImageChops
-from playwright.sync_api import sync_playwright
 import pytest
 from IPython.core.oinspect import Inspector, object_info
 
@@ -212,7 +209,7 @@ def _set_docrepr_options(**docrepr_options):
     ids=list(TEST_CASES.keys()),
     )
 def test_sphinxify(
-        build_oinfo, set_docrepr_options, open_browser,
+        build_oinfo, set_docrepr_options, open_browser, compare_screenshots,
         test_id, obj, oinfo_data, docrepr_options,
         ):
     if (oinfo_data.get("docstring", None) == PLOT_DOCSTRING
@@ -222,14 +219,6 @@ def test_sphinxify(
         pytest.skip(
             "Plot fails on Py3.6 on Windows; older version of Matplotlib?")
 
-    # Filtering warnings generated by playwright
-    warnings.filterwarnings(
-        'ignore', category=DeprecationWarning, module='pyee.*')
-    warnings.filterwarnings(
-        'ignore', category=ResourceWarning, module='subprocess.*')
-    warnings.filterwarnings(
-        'ignore', category=ResourceWarning, module='asyncio.*')
-
     oinfo = build_oinfo(obj, **oinfo_data)
     set_docrepr_options(**docrepr_options)
 
@@ -242,35 +231,5 @@ def test_sphinxify(
     file_text = output_file.read_text(encoding='utf-8', errors='strict')
     assert len(file_text) > 512
 
-    test_dir = Path(__file__).parent
-    image = f'test-{test_id}.png'
-    reference = (test_dir / 'references' / image).resolve()
-    screenshot = (test_dir / 'screenshots' / image).resolve()
-    diff = (test_dir / 'diffs' / image).resolve()
-
-    # Create diff directory
-    (test_dir / 'diffs').mkdir(parents=True, exist_ok=True)
-
-    # Take a screenshot of the generated HTML
-    with sync_playwright() as p:
-        browser = p.chromium.launch()
-        page = browser.new_page()
-        page.goto(f'file://{url}')
-
-        # Wait for mathjax to finish rendering
-        page.wait_for_selector('#MathJax_Message', state='hidden')
-
-        page.screenshot(path=screenshot)
-        browser.close()
-
-    # Compare the screenshot with the reference
-    reference_im = Image.open(reference).convert('RGB')
-    screenshot_im = Image.open(screenshot).convert('RGB')
-    diff_im = ImageChops.difference(screenshot_im, reference_im)
-
-    bbox = diff_im.getbbox()
-    if bbox is not None:
-        diff_im.save(diff)
-
-    assert bbox is None, \
-        f'{test_id} screenshot and reference do not match'
+    compare_screenshots(test_id, url)
+    open_browser(url)