Visual Regression in CI

Build a reliable visual regression pipeline — Docker-based baselines, update workflows, and PR gates that don't produce false positives.

The CI Visual Testing Problem

Visual tests are notoriously flaky in CI because:

1. Platform differences — Linux CI renders fonts differently than macOS dev machines
2. Timing issues — animations, lazy images, fonts not loaded at capture time
3. Dynamic content — timestamps, ads, user-specific data change every run
4. Baseline drift — baselines committed by one machine fail on another

This page covers strategies to make visual CI reliable.

Strategy 1: Generate Baselines in Docker

Ensure baselines and test runs use identical rendering environments:

# Dockerfile.visual — use the official Playwright image
FROM mcr.microsoft.com/playwright:v1.45.0-jammy

WORKDIR /app
COPY package*.json ./
RUN npm ci
COPY . .

# Generate baselines (run once, commit the output)
docker run --rm -v $(pwd):/app playwright-visual \
  npx playwright test visual.spec.ts --update-snapshots

# Verify baselines (every CI run)
docker run --rm -v $(pwd):/app playwright-visual \
  npx playwright test visual.spec.ts

# .github/workflows/visual.yml
jobs:
  visual:
    runs-on: ubuntu-latest
    container:
      image: mcr.microsoft.com/playwright:v1.45.0-jammy
    steps:
      - uses: actions/checkout@v4
      - run: npm ci
      - run: npx playwright test visual.spec.ts
      - uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: visual-diff
          path: test-results/

Strategy 2: Stabilise Before Capture

// fixtures/stable-page.fixture.ts
import { test as base, Page } from '@playwright/test';

async function stabilisePage(page: Page) {
  // Wait for network to settle
  await page.waitForLoadState('networkidle');

  // Freeze animations and transitions
  await page.addStyleTag({
    content: `
      *, *::before, *::after {
        animation-duration: 0s !important;
        animation-delay: 0s !important;
        transition-duration: 0s !important;
        transition-delay: 0s !important;
      }
    `,
  });

  // Wait for custom fonts to load
  await page.evaluate(() => document.fonts.ready);

  // Wait for lazy images
  await page.evaluate(() =>
    Promise.all(
      Array.from(document.querySelectorAll('img[loading="lazy"]'))
        .filter((img): img is HTMLImageElement => img instanceof HTMLImageElement)
        .map(img => img.complete ? Promise.resolve() : new Promise(r => { img.onload = r; }))
    )
  );
}

export const test = base.extend({
  page: async ({ page }, use) => {
    // Expose stabilise as a method
    (page as any).stabilise = () => stabilisePage(page);
    await use(page);
  },
});

Strategy 3: Update Snapshot Workflow

When intentional visual changes land, baselines need updating. Automate this:

# .github/workflows/update-snapshots.yml
name: Update Visual Snapshots
on:
  workflow_dispatch:    # manual trigger
    inputs:
      test_filter:
        description: 'Test file or grep pattern (optional)'
        required: false

jobs:
  update:
    runs-on: ubuntu-latest
    container:
      image: mcr.microsoft.com/playwright:v1.45.0-jammy
    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - run: npm ci
      - run: npx playwright test ${{ inputs.test_filter }} --update-snapshots
      - name: Commit updated snapshots
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add '**/*-snapshots/**'
          git diff --staged --quiet || git commit -m "chore: update visual snapshots"
          git push

Strategy 4: Visual Tests as Non-Blocking Gates

For teams starting with visual testing, run visual tests in informational mode:

# Don't fail the PR, but report diffs
- name: Visual tests (informational)
  run: npx playwright test visual.spec.ts || true  # never fails CI
  
- name: Upload diff report
  uses: actions/upload-artifact@v4
  with:
    name: visual-diff-report
    path: playwright-report/

Upgrade to blocking once baselines are stable and the team trusts the suite.

Visual Test File Organisation

// Separate visual tests from functional tests
e2e/
├── functional/          // fast, always blocking
│   ├── auth.spec.ts
│   └── checkout.spec.ts
└── visual/              // slower, blocking after stabilisation
    ├── pages.spec.ts    // full-page snapshots
    └── components.spec.ts  // component-level snapshots

// playwright.config.ts
projects: [
  { name: 'functional', testDir: 'e2e/functional' },
  {
    name: 'visual',
    testDir: 'e2e/visual',
    use: { viewport: { width: 1280, height: 720 } },
  },
]