Visual regression tests with playwright

.github/pull_request_template.md

@@ -0,0 +1,9 @@
+<!--
+## Visual changes
+
+If this PR has an impact on the rendered output, you can ask the bot to
+update the reference screenshots for the visual regression tests by
+commenting the following:
+
+"Please update reference screenshots"
+-->

.github/workflows/test.yaml

@@ -22,6 +22,11 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ['3.6', '3.10']
+        include:
+        - python-version: '3.6'
+          pytest-args: ''
+        - python-version: '3.10'
+          pytest-args: '--asyncio-mode=auto'
     steps:
     - name: Checkout branch
       uses: actions/checkout@v2
@@ -47,7 +52,7 @@ jobs:
       run: pip list
     - name: Run tests
       shell: bash
-      run: python -I -bb -X dev -W error -m pytest
+      run: python -I -bb -X dev -W error -m pytest ${{ matrix.pytest-args }}
     - name: Twine check
       run: pipx run twine check --strict dist/*
     - name: Pip check

.github/workflows/update_reference_screenshots.yaml

@@ -0,0 +1,67 @@
+name: Update Reference Screenshots
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+
+jobs:
+  update-reference-screenshots:
+    name: Update reference screenshots on ${{ matrix.os }}
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'Please update reference screenshots') }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        include:
+        - os: ubuntu-latest
+          install-command: 'sudo apt'
+        - os: windows-latest
+          install-command: 'choco'
+        - os: macos-latest
+          install-command: 'brew'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Checkout the branch from the PR that triggered the job
+        run: hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Print basic Python info
+        shell: python
+        run: |
+          import sys
+          print(sys.executable, sys.version, sep='\n')
+      - name: Install docrepr
+        run: python -m pip install --upgrade .[test,visual_test]
+      - name: List packages in environment
+        run: pip list
+      - name: Install Firefox
+        run: python -m playwright install firefox
+      - name: Generate updated reference screenshots
+        shell: bash
+        run: python -I -bb -X dev -W error -m pytest --update-reference-screenshots --asyncio-mode=auto
+      - name: Compress screenshots
+        shell: bash
+        run: |
+          ${{ matrix.install-command }} install optipng
+          optipng docrepr/tests/reference_screenshots/*.png
+      - name: Commit reference images
+        run: |
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+
+          git pull
+
+          git add docrepr/tests/reference_screenshots
+
+          git commit -m "Update reference screenshots for ${{ matrix.os }}"
+          git push
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/visual_regression_test.yaml

@@ -0,0 +1,53 @@
+# Run the project's test suite
+name: Visual Regression Tests
+
+on:
+  push:
+    branches:
+    - master
+    - main
+    - '*.x'
+  pull_request:
+    branches:
+    - master
+    - main
+    - '*.x'
+
+jobs:
+  visual-regression-test:
+    name: Visual Regression Tests ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+    - name: Checkout branch
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Print basic Python info
+      shell: python
+      run: |
+        import sys
+        print(sys.executable, sys.version, sep='\n')
+    - name: Install docrepr
+      run: python -m pip install --upgrade .[test,visual_test]
+    - name: List packages in environment
+      run: pip list
+    - name: Install Firefox
+      run: python -m playwright install firefox
+    - name: Run tests
+      shell: bash
+      run: python -I -bb -X dev -W error -m pytest --compare-screenshots --asyncio-mode=auto
+    - name: Upload UI test artifacts
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: visual-regression-test-output
+        path: |
+          docrepr/tests/reference_screenshots/**
+          docrepr/tests/screenshots/**
+          docrepr/tests/diffs/**

.gitignore

@@ -72,6 +72,8 @@ pip-delete-this-directory.txt
 coverage.xml
 htmlcov/
 nosetests.xml
+docrepr/tests/screenshots/
+docrepr/tests/diffs/
 
 # Translations
 *.mo

conftest.py

@@ -1,28 +1,49 @@
 """Setup for Pytest."""
 
 # Standard library imports
+import shutil
+import sys
 import warnings
 import webbrowser
+from pathlib import Path
 
 # Third party imports
 import pytest
-
+from PIL import Image, ImageChops
 
 # ---- Constants
 
+TEST_DIR = Path(__file__).parent / 'docrepr' / 'tests'
+
 OPEN_BROWSER_OPTION = '--open-browser'
+COMPARE_SCREENSHOTS_OPTION = '--compare-screenshots'
+UPDATE_REFERENCE_SCREENSHOTS_OPTION = '--update-reference-screenshots'
 
 
 # ---- Pytest hooks
 
 def pytest_addoption(parser):
-    """Add an option to open the user's web browser with HTML test output."""
+    """Add command line options."""
     parser.addoption(
         OPEN_BROWSER_OPTION,
         action='store_true',
         default=False,
         help='For tests that generate HTML output, open it in a web browser',
     )
+    parser.addoption(
+        COMPARE_SCREENSHOTS_OPTION,
+        action='store_true',
+        default=False,
+        help=('For tests that generate HTML output, '
+              'run visual regression tests on it'),
+    )
+    parser.addoption(
+        UPDATE_REFERENCE_SCREENSHOTS_OPTION,
+        action='store_true',
+        default=False,
+        help=('For tests that generate HTML output, '
+              'update reference screenshots for the visual regression tests'),
+    )
 
 
 # ---- Fixtures
@@ -37,3 +58,60 @@ def _open_browser(url):
 
             webbrowser.open_new_tab(url)
     return _open_browser
+
+
+@pytest.fixture
+async def compare_screenshots(request):
+    """Run visual regression test on the output."""
+    async def _compare_screenshots(test_id, url):
+        if (request.config.getoption(COMPARE_SCREENSHOTS_OPTION) or
+           request.config.getoption(UPDATE_REFERENCE_SCREENSHOTS_OPTION)):
+            from playwright.async_api import async_playwright
+
+            # Filter warnings generated by playwright
+            warnings.filterwarnings(
+                'ignore', category=DeprecationWarning, module='pyee.*')
+
+            image = f'test-{test_id}-{sys.platform}.png'
+            reference = (TEST_DIR / 'reference_screenshots' / image).resolve()
+            screenshot = (TEST_DIR / 'screenshots' / image).resolve()
+            diff = (TEST_DIR / 'diffs' / image).resolve()
+
+            # Create directories
+            (TEST_DIR / 'screenshots').mkdir(parents=True, exist_ok=True)
+            (TEST_DIR / 'diffs').mkdir(parents=True, exist_ok=True)
+
+            # Take a screenshot of the generated HTML
+            async with async_playwright() as p:
+                browser = await p.firefox.launch()
+                page = await browser.new_page()
+                await page.goto(f'file://{url}', wait_until='networkidle')
+
+                # Wait for mathjax to finish rendering
+                await page.wait_for_selector(
+                    '#MathJax_Message', state='hidden')
+
+                await page.screenshot(path=screenshot)
+                await browser.close()
+
+            # Compress the screenshot
+            screenshot_im = Image.open(screenshot).convert('RGB')
+            screenshot_im.save(screenshot, optimize=True, quality=70)
+
+            # If a reference update has been requested, we copy the screenshot
+            # and consider the test as "passing"
+            if request.config.getoption(UPDATE_REFERENCE_SCREENSHOTS_OPTION):
+                shutil.copyfile(screenshot, reference)
+                return
+
+            # Compare the screenshot with the reference
+            reference_im = Image.open(reference).convert('RGB')
+            diff_im = ImageChops.difference(screenshot_im, reference_im)
+
+            bbox = diff_im.getbbox()
+            if bbox is not None:
+                diff_im.save(diff)
+
+            assert bbox is None, \
+                f'{test_id} screenshot and reference do not match'
+    return _compare_screenshots

docrepr/tests/test_output.py

@@ -166,7 +166,10 @@ def eat_one(self):
 # ---- Helper functions
 
 def _test_cases_to_params(test_cases):
-    return [tuple(test_case.values()) for test_case in test_cases.values()]
+    return [
+        (test_id, *test_case.values())
+        for test_id, test_case in test_cases.items()
+    ]
 
 
 # ---- Fixtures
@@ -199,14 +202,15 @@ def _set_docrepr_options(**docrepr_options):
 
 # ---- Tests
 
+@pytest.mark.asyncio
 @pytest.mark.parametrize(
-    ('obj', 'oinfo_data', 'docrepr_options'),
+    ('test_id', 'obj', 'oinfo_data', 'docrepr_options'),
     _test_cases_to_params(TEST_CASES),
     ids=list(TEST_CASES.keys()),
     )
-def test_sphinxify(
-        build_oinfo, set_docrepr_options, open_browser,
-        obj, oinfo_data, docrepr_options,
+async def test_sphinxify(
+        build_oinfo, set_docrepr_options, open_browser, compare_screenshots,
+        test_id, obj, oinfo_data, docrepr_options,
         ):
     if (oinfo_data.get("docstring", None) == PLOT_DOCSTRING
             and sys.version_info.major == 3
@@ -227,4 +231,5 @@ def test_sphinxify(
     file_text = output_file.read_text(encoding='utf-8', errors='strict')
     assert len(file_text) > 512
 
+    await compare_screenshots(test_id, url)
     open_browser(url)

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --durations=5 -v -r a --color=yes --code-highlight=yes --strict-config --strict-markers --import-mode=importlib --maxfail 5
+addopts = --durations=5 -v -r a --color=yes --code-highlight=yes --strict-config --strict-markers --import-mode=importlib
 empty_parameter_set_mark = fail_at_collect
 filterwarnings =
     error

setup.cfg

@@ -62,3 +62,6 @@ test =
     matplotlib>=2.2.4
     numpy
     pytest>=6.0.0
+    pytest-asyncio
+visual_test =
+    playwright