Visual regression tests with playwright

.github/workflows/test.yaml

@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.6', '3.10']
+        python-version: ['3.7', '3.10']
     steps:
     - name: Checkout branch
       uses: actions/checkout@v2
@@ -45,9 +45,20 @@ jobs:
       run: 'echo dist/*.whl | xargs -I % python -bb -X dev -W error -W ignore::DeprecationWarning:pip._internal.locations._distutils -W ignore::DeprecationWarning:distutils.command.install -m pip install --upgrade %[test]'
     - name: List packages in environment
       run: pip list
+    - name: Install Chromium
+      run: python -m playwright install chromium
     - name: Run tests
       shell: bash
       run: python -I -bb -X dev -W error -m pytest
+    - name: Upload UI Test artifacts
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: visual-regression-test-output-${{ matrix.os }}
+        path: |
+          docrepr/tests/references/**
+          docrepr/tests/screenshots/**
+          docrepr/tests/diffs/**
     - name: Twine check
       run: pipx run twine check --strict dist/*
     - name: Pip check

.gitignore

@@ -72,6 +72,8 @@ pip-delete-this-directory.txt
 coverage.xml
 htmlcov/
 nosetests.xml
+docrepr/tests/screenshots/**
+docrepr/tests/diffs/**
 
 # Translations
 *.mo

docrepr/tests/test_output.py

@@ -4,10 +4,14 @@
 import copy
 import subprocess
 import sys
+from time import sleep
 from pathlib import Path
+import warnings
 
 # Third party imports
 import numpy as np
+from PIL import Image, ImageChops
+from playwright.sync_api import sync_playwright
 import pytest
 from IPython.core.oinspect import Inspector, object_info
 
@@ -97,7 +101,10 @@
 # ---- Helper functions
 
 def _test_cases_to_params(test_cases):
-    return [tuple(test_case.values()) for test_case in test_cases.values()]
+    return [
+        [test_id, *test_case.values()]
+        for test_id, test_case in test_cases.items()
+    ]
 
 
 # ---- Fixtures
@@ -131,13 +138,13 @@ def _set_docrepr_options(**docrepr_options):
 # ---- Tests
 
 @pytest.mark.parametrize(
-    ('obj', 'oinfo_data', 'docrepr_options'),
+    ('test_id', 'obj', 'oinfo_data', 'docrepr_options'),
     _test_cases_to_params(TEST_CASES),
     ids=list(TEST_CASES.keys()),
     )
 def test_sphinxify(
         build_oinfo, set_docrepr_options, open_browser,
-        obj, oinfo_data, docrepr_options,
+        test_id, obj, oinfo_data, docrepr_options,
         ):
     if (oinfo_data.get("docstring", None) == PLOT_DOCSTRING
             and sys.version_info.major == 3
@@ -146,6 +153,14 @@ def test_sphinxify(
         pytest.skip(
             "Plot fails on Py3.6 on Windows; older version of Matplotlib?")
 
+    # Filtering warnings generated by playwright
+    warnings.filterwarnings(
+        'ignore', category=DeprecationWarning, module='pyee.*')
+    warnings.filterwarnings(
+        'ignore', category=ResourceWarning, module='subprocess.*')
+    warnings.filterwarnings(
+        'ignore', category=ResourceWarning, module='asyncio.*')
+
     oinfo = build_oinfo(obj, **oinfo_data)
     set_docrepr_options(**docrepr_options)
 
@@ -158,4 +173,35 @@ def test_sphinxify(
     file_text = output_file.read_text(encoding='utf-8', errors='strict')
     assert len(file_text) > 512
 
-    open_browser(url)
+    test_dir = Path(__file__).parent
+    image = f'test-{test_id}.png'
+    reference = (test_dir / 'references' / image).resolve()
+    screenshot = (test_dir / 'screenshots' / image).resolve()
+    diff = (test_dir / 'diffs' / image).resolve()
+
+    # Create diff directory
+    (test_dir / 'diffs').mkdir(parents=True, exist_ok=True)
+
+    # Take a screenshot of the generated HTML
+    with sync_playwright() as p:
+        browser = p.chromium.launch()
+        page = browser.new_page()
+        page.goto(f'file://{url}')
+
+        # Wait for mathjax to finish rendering
+        page.wait_for_selector('#MathJax_Message', state='hidden')
+
+        page.screenshot(path=screenshot)
+        browser.close()
+
+    # Compare the screenshot with the reference
+    reference_im = Image.open(reference).convert('RGB')
+    screenshot_im = Image.open(screenshot).convert('RGB')
+    diff_im = ImageChops.difference(screenshot_im, reference_im)
+
+    bbox = diff_im.getbbox()
+    if bbox is not None:
+        diff_im.save(diff)
+
+    assert bbox is None, \
+        f'{test_id} screenshot and reference do not match'

setup.cfg

@@ -62,3 +62,4 @@ test =
     matplotlib>=2.2.4
     numpy
     pytest>=6.0.0
+    playwright