Visual regression tests with playwright

.github/workflows/test.yaml

@@ -52,3 +52,41 @@ jobs:
       run: pipx run twine check --strict dist/*
     - name: Pip check
       run: pip check -v
+
+  visual-regression-tests:
+    name: Visual Regression Tests ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+    - name: Checkout branch
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Print basic Python info
+      shell: python
+      run: |
+        import sys
+        print(sys.executable, sys.version, sep='\n')
+    - name: Install docrepr
+      run: python -m pip install --upgrade .[test,visual_tests]
+    - name: List packages in environment
+      run: pip list
+    - name: Install Firefox
+      run: python -m playwright install firefox
+    - name: Run tests
+      shell: bash
+      run: python -I -bb -X dev -W error -m pytest --compare-screenshots
+    - name: Upload UI Test artifacts
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: visual-regression-test-output
+        path: |
+          docrepr/tests/references/**
+          docrepr/tests/screenshots/**
+          docrepr/tests/diffs/**

.github/workflows/update_references.yaml

@@ -0,0 +1,56 @@
+name: Update references
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+
+jobs:
+  update_references:
+
+    name: PR comment
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'Please update reference screenshots') }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Checkout the branch from the PR that triggered the job
+        run: hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Print basic Python info
+        shell: python
+        run: |
+          import sys
+          print(sys.executable, sys.version, sep='\n')
+      - name: Install docrepr
+        run: python -m pip install --upgrade .[test,visual_tests]
+      - name: List packages in environment
+        run: pip list
+      - name: Install Firefox
+        run: python -m playwright install firefox
+      - name: Run tests
+        shell: bash
+        run: python -I -bb -X dev -W error -m pytest --update-reference-screenshots
+      - name: Commit reference images
+        run: |
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+
+          git pull
+
+          git add docrepr/tests/references
+
+          git commit -m "Update references for ${{ matrix.os }}"
+          git push
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -72,6 +72,8 @@ pip-delete-this-directory.txt
 coverage.xml
 htmlcov/
 nosetests.xml
+docrepr/tests/screenshots/**
+docrepr/tests/diffs/**
 
 # Translations
 *.mo

conftest.py

@@ -1,28 +1,45 @@
 """Setup for Pytest."""
 
 # Standard library imports
+from pathlib import Path
+import shutil
+from sys import platform
 import warnings
 import webbrowser
 
 # Third party imports
 import pytest
-
+from PIL import Image, ImageChops
 
 # ---- Constants
 
 OPEN_BROWSER_OPTION = '--open-browser'
+COMPARE_SCREENSHOTS_OPTION = "--compare-screenshots"
+UPDATE_REFERENCE_SCREENSHOTS_OPTION = "--update-reference-screenshots"
 
 
 # ---- Pytest hooks
 
 def pytest_addoption(parser):
-    """Add an option to open the user's web browser with HTML test output."""
+    """Add command line options."""
     parser.addoption(
         OPEN_BROWSER_OPTION,
         action='store_true',
         default=False,
         help='For tests that generate HTML output, open it in a web browser',
     )
+    parser.addoption(
+        COMPARE_SCREENSHOTS_OPTION,
+        action='store_true',
+        default=False,
+        help='For tests that generate HTML output, run visual regression tests on them',
+    )
+    parser.addoption(
+        UPDATE_REFERENCE_SCREENSHOTS_OPTION,
+        action='store_true',
+        default=False,
+        help='For tests that generate HTML output, update reference screenshots for the visual regression tests',
+    )
 
 
 # ---- Fixtures
@@ -37,3 +54,59 @@ def _open_browser(url):
 
             webbrowser.open_new_tab(url)
     return _open_browser
+
+
+@pytest.fixture
+async def compare_screenshots(request):
+    """Run visual regression test on the output."""
+    async def _compare_screenshots(test_id, url):
+        if (request.config.getoption(COMPARE_SCREENSHOTS_OPTION) or
+            request.config.getoption(UPDATE_REFERENCE_SCREENSHOTS_OPTION)):
+            from playwright.async_api import async_playwright
+
+            # Filtering warnings generated by playwright
+            warnings.filterwarnings(
+                'ignore', category=DeprecationWarning, module='pyee.*')
+
+            test_dir = Path(__file__).parent / 'docrepr' / 'tests'
+            image = f'test-{test_id}-{platform}.png'
+            reference = (test_dir / 'references' / image).resolve()
+            screenshot = (test_dir / 'screenshots' / image).resolve()
+            diff = (test_dir / 'diffs' / image).resolve()
+
+            # Create diff directory
+            (test_dir / 'diffs').mkdir(parents=True, exist_ok=True)
+
+            # Take a screenshot of the generated HTML
+            async with async_playwright() as p:
+                browser = await p.firefox.launch()
+                page = await browser.new_page()
+                await page.goto(f'file://{url}')
+
+                # Wait for mathjax to finish rendering
+                await page.wait_for_selector('#MathJax_Message', state='hidden')
+
+                await page.screenshot(path=screenshot)
+                await browser.close()
+
+            # Compress the screenshot
+            screenshot_im = Image.open(screenshot).convert('RGB')
+            screenshot_im.save(screenshot, quality=80, optimize=True)
+
+            # If a reference update has been requested, we copy the screenshot
+            # and consider the test as "passing"
+            if request.config.getoption(UPDATE_REFERENCE_SCREENSHOTS_OPTION):
+                shutil.copyfile(screenshot, reference)
+                return
+
+            # Compare the screenshot with the reference
+            reference_im = Image.open(reference).convert('RGB')
+            diff_im = ImageChops.difference(screenshot_im, reference_im)
+
+            bbox = diff_im.getbbox()
+            if bbox is not None:
+                diff_im.save(diff)
+
+            assert bbox is None, \
+                f'{test_id} screenshot and reference do not match'
+    return _compare_screenshots

docrepr/tests/test_output.py

@@ -4,6 +4,7 @@
 import copy
 import subprocess
 import sys
+from time import sleep
 from pathlib import Path
 
 # Third party imports
@@ -166,7 +167,10 @@ def eat_one(self):
 # ---- Helper functions
 
 def _test_cases_to_params(test_cases):
-    return [tuple(test_case.values()) for test_case in test_cases.values()]
+    return [
+        [test_id, *test_case.values()]
+        for test_id, test_case in test_cases.items()
+    ]
 
 
 # ---- Fixtures
@@ -200,13 +204,13 @@ def _set_docrepr_options(**docrepr_options):
 # ---- Tests
 
 @pytest.mark.parametrize(
-    ('obj', 'oinfo_data', 'docrepr_options'),
+    ('test_id', 'obj', 'oinfo_data', 'docrepr_options'),
     _test_cases_to_params(TEST_CASES),
     ids=list(TEST_CASES.keys()),
     )
-def test_sphinxify(
-        build_oinfo, set_docrepr_options, open_browser,
-        obj, oinfo_data, docrepr_options,
+async def test_sphinxify(
+        build_oinfo, set_docrepr_options, open_browser, compare_screenshots,
+        test_id, obj, oinfo_data, docrepr_options,
         ):
     if (oinfo_data.get("docstring", None) == PLOT_DOCSTRING
             and sys.version_info.major == 3
@@ -227,4 +231,5 @@ def test_sphinxify(
     file_text = output_file.read_text(encoding='utf-8', errors='strict')
     assert len(file_text) > 512
 
+    await compare_screenshots(test_id, url)
     open_browser(url)

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --durations=5 -v -r a --color=yes --code-highlight=yes --strict-config --strict-markers --import-mode=importlib --maxfail 5
+addopts = --durations=5 -v -r a --color=yes --code-highlight=yes --strict-config --strict-markers --import-mode=importlib
 empty_parameter_set_mark = fail_at_collect
 filterwarnings =
     error
@@ -9,3 +9,4 @@ minversion = 6.0
 testpaths =
     docrepr/tests
 xfail_strict = True
+asyncio_mode = auto

setup.cfg

@@ -62,3 +62,6 @@ test =
     matplotlib>=2.2.4
     numpy
     pytest>=6.0.0
+    pytest-asyncio
+visual_tests =
+    playwright