From 705dc76ad896790f407ff46924093e7d1f0e3414 Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Thu, 15 Oct 2026 15:52:26 +0100 Subject: [PATCH 1/2] Add visual regression testing for RNTester Adds screenshot comparison for RNTester in public CI. An untrusted workflow captures cropped screenshots with Maestro, and a trusted workflow running from main compares them with baselines stored in a GCS bucket, reports on the PR, and publishes new baselines on pushes to main. Maintainers approve intentional changes with the visual-change-approved label. Changelog: [Internal] --- .../workflow-scripts/__tests__/visual-test.js | 622 ++++++++++++++++++ .github/workflow-scripts/visual/approval.js | 84 +++ .../workflow-scripts/visual/approve-run.js | 182 +++++ .github/workflow-scripts/visual/capture.js | 205 ++++++ .../workflow-scripts/visual/compare-run.js | 441 +++++++++++++ .github/workflow-scripts/visual/compare.js | 152 +++++ .github/workflow-scripts/visual/github.js | 145 ++++ .github/workflow-scripts/visual/manifest.js | 267 ++++++++ .github/workflow-scripts/visual/png.js | 30 + .github/workflow-scripts/visual/report.js | 217 ++++++ .github/workflow-scripts/visual/storage.js | 140 ++++ .github/workflow-scripts/visual/validate.js | 183 ++++++ .github/workflows/visual-approve-rntester.yml | 67 ++ .github/workflows/visual-capture-rntester.yml | 201 ++++++ .github/workflows/visual-compare-rntester.yml | 99 +++ package.json | 2 + .../js/examples/Text/TextSharedExamples.js | 2 +- packages/rn-tester/visual-tests/config.json | 8 + .../flows/text-last-line-clipping.yml | 15 + .../visual-tests/flows/text-width-mode.yml | 18 + .../visual-tests/helpers/open-example.yml | 15 + yarn.lock | 17 + 22 files changed, 3111 insertions(+), 1 deletion(-) create mode 100644 .github/workflow-scripts/__tests__/visual-test.js create mode 100644 .github/workflow-scripts/visual/approval.js create mode 100644 .github/workflow-scripts/visual/approve-run.js create mode 100644 .github/workflow-scripts/visual/capture.js create mode 100644 .github/workflow-scripts/visual/compare-run.js create mode 100644 .github/workflow-scripts/visual/compare.js create mode 100644 .github/workflow-scripts/visual/github.js create mode 100644 .github/workflow-scripts/visual/manifest.js create mode 100644 .github/workflow-scripts/visual/png.js create mode 100644 .github/workflow-scripts/visual/report.js create mode 100644 .github/workflow-scripts/visual/storage.js create mode 100644 .github/workflow-scripts/visual/validate.js create mode 100644 .github/workflows/visual-approve-rntester.yml create mode 100644 .github/workflows/visual-capture-rntester.yml create mode 100644 .github/workflows/visual-compare-rntester.yml create mode 100644 packages/rn-tester/visual-tests/config.json create mode 100644 packages/rn-tester/visual-tests/flows/text-last-line-clipping.yml create mode 100644 packages/rn-tester/visual-tests/flows/text-width-mode.yml create mode 100644 packages/rn-tester/visual-tests/helpers/open-example.yml diff --git a/.github/workflow-scripts/__tests__/visual-test.js b/.github/workflow-scripts/__tests__/visual-test.js new file mode 100644 index 000000000000..bcecae2a80dc --- /dev/null +++ b/.github/workflow-scripts/__tests__/visual-test.js @@ -0,0 +1,622 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + APPROVAL_LABEL, + differenceFingerprint, + gateDecision, +} = require('../visual/approval'); +const { + collectFlows, + failedFlowsFromJUnit, + maestroArgs, +} = require('../visual/capture'); +const {compareCapture, compareImages, policyFor} = require('../visual/compare'); +const {buildManifest, environmentKey, sha256} = require('../visual/manifest'); +const {readPngDimensions} = require('../visual/png'); +const { + COMMENT_MARKER, + renderCheckOutput, + renderPullRequestComment, +} = require('../visual/report'); +const { + approvalKey, + baselinePrefix, + publicUrl, + reportPrefix, +} = require('../visual/storage'); +const {validateCapture} = require('../visual/validate'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const {PNG} = require('pngjs'); + +const SHA_A = 'a'.repeat(40); +const SHA_B = 'b'.repeat(40); + +const ANDROID_ENVIRONMENT = { + apiLevel: 35, + abi: 'x86_64', + nativeBridge: 'libndk_translation.so', + systemImage: + 'google/sdk_gphone64_x86_64/emu64xa:15/AE3A/1:userdebug/dev-keys', + screenSize: '1080x2400', + density: 420, + fontScale: '1.0', + locale: 'en-US', + animatorDurationScale: '0.0', +}; + +function makePng(width, height, paint = () => [255, 255, 255, 255]) { + const png = new PNG({width, height}); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const offset = (y * width + x) * 4; + const [r, g, b, a] = paint(x, y); + png.data[offset] = r; + png.data[offset + 1] = g; + png.data[offset + 2] = b; + png.data[offset + 3] = a; + } + } + return PNG.sync.write(png); +} + +function writeCapture(directory, images, overrides = {}) { + for (const [id, buffer] of Object.entries(images)) { + const filePath = path.join(directory, `${id}.png`); + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + fs.writeFileSync(filePath, buffer); + } + const manifest = buildManifest({ + captureDir: directory, + platform: 'android', + environment: ANDROID_ENVIRONMENT, + git: {testedSha: SHA_A, baseSha: SHA_B, headSha: SHA_A}, + }); + const finalManifest = {...manifest, ...overrides}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(finalManifest), + ); + return finalManifest; +} + +describe('visual regression', () => { + let directory; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-test-')); + }); + + afterEach(() => { + fs.rmSync(directory, {recursive: true, force: true}); + }); + + describe('readPngDimensions', () => { + test('reads the IHDR dimensions', () => { + expect(readPngDimensions(makePng(3, 2))).toEqual({width: 3, height: 2}); + }); + + test.each([ + [Buffer.from('not a png at all, but long enough')], + [Buffer.alloc(0)], + ])('rejects non-PNG data', buffer => { + expect(readPngDimensions(buffer)).toBeNull(); + }); + }); + + describe('environmentKey', () => { + test('is readable and stable', () => { + expect(environmentKey('android', ANDROID_ENVIRONMENT)).toMatch( + /^android-api35-x86-64-1080x2400-420dpi-[0-9a-f]{8}$/, + ); + expect(environmentKey('android', {...ANDROID_ENVIRONMENT})).toBe( + environmentKey('android', ANDROID_ENVIRONMENT), + ); + }); + + test('changes when any keyed property drifts', () => { + expect( + environmentKey('android', {...ANDROID_ENVIRONMENT, fontScale: '1.15'}), + ).not.toBe(environmentKey('android', ANDROID_ENVIRONMENT)); + }); + + test('ignores properties that are only recorded', () => { + expect( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + appearance: 'light', + contentSizeCategory: 'large', + locale: 'en_US', + xcode: 'Xcode 26.2', + }), + ).toBe( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + appearance: 'light', + contentSizeCategory: 'large', + locale: 'en_US', + xcode: 'Xcode 26.3', + }), + ); + }); + test('names iOS keys by runtime and device', () => { + expect( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + }), + ).toMatch(/^ios-26-2-iphone-17-pro-[0-9a-f]{8}$/); + }); + }); + + describe('validateCapture', () => { + test('accepts a well-formed capture', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + expect(validateCapture(directory)).toEqual(manifest); + }); + + test.each([ + ['../escape/a/b'], + ['/absolute/a/b'], + ['Text/Upper/case'], + ['too/few'], + ['a/b/c/d'], + ])('rejects test ID %s', id => { + writeCapture(directory, {'text/width-mode/default': makePng(4, 4)}); + const manifest = JSON.parse( + fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'), + ); + manifest.images[0] = {...manifest.images[0], id, path: `${id}.png`}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + expect(() => validateCapture(directory)).toThrow('Invalid test ID'); + }); + + test('rejects a path that does not match the ID', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + manifest.images[0].path = '../../etc/passwd'; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + expect(() => validateCapture(directory)).toThrow('path must be'); + }); + + test('rejects duplicates, mismatched hashes and unlisted files', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + manifest.images.push({...manifest.images[0]}); + manifest.images[0] = {...manifest.images[0], sha256: '0'.repeat(64)}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + fs.writeFileSync(path.join(directory, 'extra.sh'), 'echo hi'); + let message = ''; + try { + validateCapture(directory); + } catch (error) { + message = error.message; + } + expect(message).toContain('Duplicate test ID'); + expect(message).toContain('does not match its manifest entry'); + expect(message).toContain('Files not in the manifest: extra.sh'); + }); + + test('rejects non-PNG content and oversized images', () => { + writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + 'text/width-mode/large': makePng(20, 4), + }); + fs.writeFileSync( + path.join(directory, 'text/width-mode/default.png'), + 'not a png', + ); + expect(() => + validateCapture(directory, { + maxImages: 10, + maxImageBytes: 1024 * 1024, + maxTotalBytes: 1024 * 1024, + maxDimension: 10, + maxManifestBytes: 1024 * 1024, + }), + ).toThrow(/default: not a PNG[\s\S]*large: 20x4 is outside/); + }); + + test('rejects symlinks', () => { + writeCapture(directory, {'text/width-mode/default': makePng(4, 4)}); + fs.rmSync(path.join(directory, 'text/width-mode/default.png')); + fs.symlinkSync( + '/etc/hosts', + path.join(directory, 'text/width-mode/default.png'), + ); + expect(() => validateCapture(directory)).toThrow( + 'file is missing or not a regular file', + ); + }); + + test('rejects invalid SHAs and environment keys', () => { + writeCapture( + directory, + {'text/width-mode/default': makePng(4, 4)}, + { + environmentKey: 'ios-../../x', + git: {testedSha: 'HEAD', baseSha: SHA_B, headSha: SHA_A}, + }, + ); + expect(() => validateCapture(directory)).toThrow( + /Invalid environment key[\s\S]*Invalid git.testedSha/, + ); + }); + }); + + describe('compareImages', () => { + const strict = policyFor('a/b/c', null); + + test('identical images are unchanged', () => { + expect(compareImages(makePng(8, 8), makePng(8, 8), strict)).toMatchObject( + {status: 'unchanged', diffPixels: 0, diff: null}, + ); + }); + + test('a single changed pixel fails the strict policy', () => { + const changed = makePng(8, 8, (x, y) => + x === 3 && y === 3 ? [254, 255, 255, 255] : [255, 255, 255, 255], + ); + const result = compareImages(makePng(8, 8), changed, strict); + expect(result).toMatchObject({status: 'changed', diffPixels: 1}); + expect(readPngDimensions(result.diff)).toEqual({width: 8, height: 8}); + }); + + test('a per-test tolerance can absorb a known amount of noise', () => { + const changed = makePng(8, 8, (x, y) => + x === 3 && y === 3 ? [0, 0, 0, 255] : [255, 255, 255, 255], + ); + const policy = policyFor('a/b/c', {tests: {'a/b/c': {maxDiffPixels: 1}}}); + expect(compareImages(makePng(8, 8), changed, policy).status).toBe( + 'unchanged', + ); + expect( + policyFor('a/b/d', {tests: {'a/b/c': {maxDiffPixels: 1}}}), + ).toEqual(strict); + }); + + test('different dimensions are reported, not compared', () => { + expect(compareImages(makePng(8, 8), makePng(8, 9), strict)).toMatchObject( + { + status: 'dimension-mismatch', + expectedSize: {width: 8, height: 8}, + capturedSize: {width: 8, height: 9}, + }, + ); + }); + }); + + describe('compareCapture', () => { + test('classifies unchanged, changed, added and removed tests', async () => { + const white = makePng(4, 4); + const black = makePng(4, 4, () => [0, 0, 0, 255]); + const image = (id, buffer) => ({id, sha256: sha256(buffer)}); + const results = await compareCapture({ + manifest: { + images: [ + image('a/a/same', white), + image('a/a/changed', black), + image('a/a/added', white), + ], + }, + baselineManifest: { + images: [ + image('a/a/same', white), + image('a/a/changed', white), + image('a/a/removed', white), + ], + }, + readExpected: async () => white, + readCaptured: async () => black, + config: null, + }); + expect(results.map(result => [result.id, result.status])).toEqual([ + ['a/a/added', 'added'], + ['a/a/changed', 'changed'], + ['a/a/removed', 'removed'], + ['a/a/same', 'unchanged'], + ]); + }); + }); + + describe('storage keys', () => { + test('builds write-once keys', () => { + expect(baselinePrefix('ios-x-0123abcd', SHA_A)).toBe( + `baselines/ios-x-0123abcd/${SHA_A}`, + ); + expect( + reportPrefix({prNumber: 12, headSha: SHA_A, runId: 34, runAttempt: 2}), + ).toBe(`reports/pull/12/${SHA_A}/34-2`); + expect( + reportPrefix({ + prNumber: null, + headSha: SHA_A, + runId: 34, + runAttempt: 1, + }), + ).toBe(`reports/main/${SHA_A}/34-1`); + expect(approvalKey(12, '0123456789abcdef')).toBe( + 'approvals/pull/12/0123456789abcdef.json', + ); + }); + + test.each([ + [() => baselinePrefix('k', 'main')], + [ + () => + reportPrefix({prNumber: -1, headSha: SHA_A, runId: 1, runAttempt: 1}), + ], + [ + () => + reportPrefix({ + prNumber: 1, + headSha: SHA_A, + runId: 1.5, + runAttempt: 1, + }), + ], + [() => approvalKey(1, '../x')], + ])('rejects invalid input', build => { + expect(build).toThrow('Invalid'); + }); + + test('encodes each path segment of public URLs', () => { + expect(publicUrl('https://cdn.example/bucket/', 'a b/c#d.png')).toBe( + 'https://cdn.example/bucket/a%20b/c%23d.png', + ); + }); + }); + + describe('approval', () => { + const platform = results => ({ + platform: 'ios', + environmentKey: 'ios-x-0123abcd', + baselineSha: SHA_B, + results, + }); + const changed = sha => ({ + id: 'a/b/c', + status: 'changed', + expected: {sha256: 'e'}, + captured: {sha256: sha}, + }); + + test('has no fingerprint without differences', () => { + expect( + differenceFingerprint([platform([{id: 'a/b/c', status: 'unchanged'}])]), + ).toBeNull(); + }); + + test('fingerprints the exact captured images', () => { + const first = differenceFingerprint([platform([changed('1')])]); + expect(first).toMatch(/^[0-9a-f]{16}$/); + expect(differenceFingerprint([platform([changed('1')])])).toBe(first); + expect(differenceFingerprint([platform([changed('2')])])).not.toBe(first); + }); + + test.each([ + [null, false, null, 'success', false], + ['f', false, null, 'failure', false], + ['f', true, {fingerprint: 'f', approver: 'x'}, 'success', false], + ['f', true, {fingerprint: 'g', approver: 'x'}, 'failure', true], + ['f', true, null, 'failure', true], + ])( + 'gate for fingerprint %s, label %s, approval %j', + (fingerprint, labelPresent, approval, conclusion, staleLabel) => { + const decision = gateDecision({fingerprint, labelPresent, approval}); + expect(decision.conclusion).toBe(conclusion); + expect(decision.staleLabel ?? false).toBe(staleLabel); + if (conclusion === 'failure' && !staleLabel) { + expect(decision.summary).toContain(APPROVAL_LABEL); + } + }, + ); + }); + + describe('capture', () => { + test('collects top-level flows in order and ignores other files', () => { + fs.writeFileSync(path.join(directory, 'b.yml'), ''); + fs.writeFileSync(path.join(directory, 'a.yaml'), ''); + fs.writeFileSync(path.join(directory, 'notes.md'), ''); + fs.mkdirSync(path.join(directory, 'helpers')); + fs.writeFileSync(path.join(directory, 'helpers/open.yml'), ''); + expect(collectFlows(directory)).toEqual([ + path.join(directory, 'a.yaml'), + path.join(directory, 'b.yml'), + ]); + }); + + test('rejects flows Maestro would report under the same name', () => { + fs.writeFileSync(path.join(directory, 'a.yml'), ''); + fs.writeFileSync(path.join(directory, 'a.yaml'), ''); + expect(() => collectFlows(directory)).toThrow("both report as 'a'"); + }); + + test('reads failed flows from a Maestro JUnit report', () => { + const flows = ['/flows/text-a.yml', '/flows/text-b.yml', '/flows/c.yml']; + const report = [ + '', + '', + '', + ].join('\n'); + expect(failedFlowsFromJUnit(report, flows)).toEqual([ + '/flows/text-b.yml', + '/flows/c.yml', + ]); + expect(() => + failedFlowsFromJUnit('', flows), + ).toThrow("unknown flow 'other'"); + expect(failedFlowsFromJUnit('', flows)).toEqual([]); + }); + + test.each([ + [1, undefined, false], + [2, 'UDID', true], + ])('builds Maestro arguments for attempt %d', (attempt, udid, reuse) => { + const args = maestroArgs({ + flows: ['/f/a.yml', '/f/b.yml'], + appId: 'com.example', + outputDir: '/out', + reportPath: '/r.xml', + udid, + attempt, + }); + expect( + args.slice(args.indexOf('test'), args.indexOf('test') + 3), + ).toEqual(['test', '/f/a.yml', '/f/b.yml']); + expect(args).toEqual( + expect.arrayContaining([ + '--format=junit', + '--output=/r.xml', + 'APP_ID=com.example', + 'VISUAL_OUTPUT_DIR=/out', + ]), + ); + expect(args.includes('--no-reinstall-driver')).toBe(reuse); + expect(args.includes('--udid=UDID')).toBe(udid != null); + }); + }); + + describe('renderPullRequestComment', () => { + const changed = id => ({ + id, + status: 'changed', + diffPixels: 3, + diffRatio: 0.03, + expected: {}, + captured: {}, + hasDiffImage: true, + }); + const platform = (name, results, error = null) => ({ + platform: name, + environmentKey: `${name}-x-0123abcd`, + baseline: {sha: SHA_B, distance: 0}, + results, + error, + checkUrl: `https://github.example/checks/${name}`, + imageUrl: (kind, id) => `https://cdn.example/${name}/${kind}/${id}.png`, + }); + + test('shows each diff with expected and captured collapsed', () => { + const body = renderPullRequestComment({ + platforms: [ + platform('android', [ + {id: 'a/b/same', status: 'unchanged'}, + changed('a/b/changed'), + ]), + platform('ios', [{id: 'a/b/new', status: 'added', captured: {}}]), + ], + headSha: SHA_A, + gateSummary: 'Needs approval.', + }); + expect(body.startsWith(COMMENT_MARKER)).toBe(true); + expect(body).toContain( + '**2 visual difference(s)** at `aaaaaaaaaaaa`. Needs approval.', + ); + expect(body).toContain( + '### android `a/b/changed`: Changed, 3 px (3.000%)', + ); + expect(body).toContain( + 'diff', + ); + expect(body).toContain( + '
Expected and Captured', + ); + expect(body).toContain('### ios `a/b/new`: New'); + expect(body).toContain('
Captured'); + expect(body).not.toContain('a/b/same'); + }); + + test('lists tests beyond the limit by name', () => { + const results = Array.from({length: 23}, (_, index) => + changed(`a/b/test-${String(index).padStart(2, '0')}`), + ); + const body = renderPullRequestComment({ + platforms: [platform('android', results)], + headSha: SHA_A, + gateSummary: '', + }); + expect(body).toContain('3 more:'); + expect(body).toContain( + '- android `a/b/test-22`: Changed, 3 px (3.000%), [check run](https://github.example/checks/android)', + ); + expect(body).not.toContain('android/diff/a/b/test-20.png'); + }); + + test('reports errors and the absence of differences', () => { + const body = renderPullRequestComment({ + platforms: [ + platform('android', [{id: 'a/b/same', status: 'unchanged'}]), + platform('ios', [], {title: 'no baseline for this environment'}), + ], + headSha: SHA_A, + gateSummary: 'No visual differences.', + }); + expect(body).toContain('No visual differences at `aaaaaaaaaaaa`.'); + expect(body).toContain( + '**ios:** [no baseline for this environment](https://github.example/checks/ios)', + ); + }); + }); + + describe('renderCheckOutput', () => { + test('links expected, captured and diff images for each difference', () => { + const output = renderCheckOutput({ + platform: 'android', + environmentKey: 'android-x-0123abcd', + baseline: {sha: SHA_B, distance: 2}, + results: [ + {id: 'a/b/same', status: 'unchanged'}, + { + id: 'a/b/changed', + status: 'changed', + diffPixels: 3, + diffRatio: 0.03, + expected: {}, + captured: {}, + hasDiffImage: true, + }, + ], + imageUrl: (kind, id) => `https://cdn.example/${kind}/${id}.png`, + error: null, + }); + expect(output.title).toBe('android: 1 visual difference(s)'); + expect(output.summary).toContain('2 commit(s) earlier'); + expect(output.text).toContain( + 'https://cdn.example/expected/a/b/changed.png', + ); + expect(output.text).toContain( + 'https://cdn.example/captured/a/b/changed.png', + ); + expect(output.text).toContain('3 px (3.000%)'); + expect(output.images).toEqual([ + expect.objectContaining({ + image_url: 'https://cdn.example/diff/a/b/changed.png', + }), + ]); + }); + }); +}); diff --git a/.github/workflow-scripts/visual/approval.js b/.github/workflow-scripts/visual/approval.js new file mode 100644 index 000000000000..c9569332f384 --- /dev/null +++ b/.github/workflow-scripts/visual/approval.js @@ -0,0 +1,84 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {sha256} = require('./manifest'); + +const APPROVAL_LABEL = 'visual-change-approved'; +const CHECK_NAME_PREFIX = 'Visual regression'; +const GATE_CHECK_NAME = 'Visual regression approval'; +const APPROVER_PERMISSIONS = ['admin', 'maintain', 'write']; + +/** + * Identifies a set of visual differences: which tests differ, how, and the + * exact expected and captured images. An approval applies to one fingerprint, + * so a push that changes any difference needs a new approval. Returns null + * when there are no differences. + */ +function differenceFingerprint(platforms) { + const lines = []; + for (const {platform, environmentKey, baselineSha, results} of platforms) { + for (const result of results) { + if (result.status === 'unchanged') { + continue; + } + lines.push( + [ + platform, + environmentKey, + baselineSha ?? '', + result.id, + result.status, + result.expected?.sha256 ?? '', + result.captured?.sha256 ?? '', + ].join('\t'), + ); + } + } + if (lines.length === 0) { + return null; + } + return sha256(Buffer.from(lines.sort().join('\n'))).slice(0, 16); +} + +/** + * The approval gate passes when there is nothing to approve, or when the + * approval label is present and an approval was recorded for this exact + * fingerprint. + */ +function gateDecision({fingerprint, labelPresent, approval}) { + if (fingerprint == null) { + return {conclusion: 'success', summary: 'No visual differences.'}; + } + if (labelPresent && approval?.fingerprint === fingerprint) { + return { + conclusion: 'success', + summary: `Visual differences approved by @${approval.approver}.`, + }; + } + if (labelPresent) { + return { + conclusion: 'failure', + summary: `The visual differences changed after approval. A maintainer must review the new report and re-apply \`${APPROVAL_LABEL}\`.`, + staleLabel: true, + }; + } + return { + conclusion: 'failure', + summary: `Visual differences need maintainer approval. Review the report, then apply \`${APPROVAL_LABEL}\`.`, + }; +} + +module.exports = { + APPROVAL_LABEL, + APPROVER_PERMISSIONS, + CHECK_NAME_PREFIX, + GATE_CHECK_NAME, + differenceFingerprint, + gateDecision, +}; diff --git a/.github/workflow-scripts/visual/approve-run.js b/.github/workflow-scripts/visual/approve-run.js new file mode 100644 index 000000000000..81a80f18b092 --- /dev/null +++ b/.github/workflow-scripts/visual/approve-run.js @@ -0,0 +1,182 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Records a maintainer's approval of a PR's visual differences and updates + * the approval gate. Runs from the default branch on `pull_request_target` + * when the approval label is added or removed. It never checks out or runs + * pull request code. + * + * node approve-run.js prepare Decide, and stage an approval record to + * upload under OUT_DIR. + * node approve-run.js report After the upload, update the gate check. + */ + +const { + APPROVAL_LABEL, + APPROVER_PERMISSIONS, + GATE_CHECK_NAME, + gateDecision, +} = require('./approval'); +const {createGitHubClient} = require('./github'); +const { + approvalKey, + fetchJsonOrNull, + listPrefixes, + publicUrl, +} = require('./storage'); +const fs = require('fs'); +const path = require('path'); + +function requireEnv(name) { + const value = process.env[name]; + if (value == null || value === '') { + throw new Error(`${name} is required`); + } + return value; +} + +/** The results of the most recent comparison for a PR head, or null. */ +async function latestResults({bucket, publicBaseUrl, prNumber, headSha}) { + const runs = await listPrefixes( + bucket, + `reports/pull/${prNumber}/${headSha}/`, + ); + const latest = runs + .map(prefix => { + const match = /\/(\d+)-(\d+)\/$/.exec(prefix); + return match == null + ? null + : {prefix, runId: Number(match[1]), attempt: Number(match[2])}; + }) + .filter(Boolean) + .sort((a, b) => b.runId - a.runId || b.attempt - a.attempt)[0]; + return latest == null + ? null + : fetchJsonOrNull(publicUrl(publicBaseUrl, `${latest.prefix}results.json`)); +} + +async function prepare() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const event = JSON.parse( + fs.readFileSync(requireEnv('GITHUB_EVENT_PATH'), 'utf8'), + ); + const writeState = state => + fs.writeFileSync( + path.join(outDir, 'state.json'), + `${JSON.stringify(state)}\n`, + ); + fs.mkdirSync(outDir, {recursive: true}); + + if (event.label?.name !== APPROVAL_LABEL) { + writeState(null); + return; + } + + const prNumber = event.pull_request.number; + const headSha = event.pull_request.head.sha; + const results = await latestResults({ + bucket: requireEnv('VISUAL_BUCKET'), + publicBaseUrl: requireEnv('VISUAL_PUBLIC_BASE_URL'), + prNumber, + headSha, + }); + const fingerprint = results?.fingerprint ?? null; + + if (event.action === 'labeled') { + const approver = event.sender.login; + const permission = await github.permission(approver); + if (!APPROVER_PERMISSIONS.includes(permission)) { + await github.removeLabel(prNumber, APPROVAL_LABEL); + await github.comment( + prNumber, + `@${approver} \`${APPROVAL_LABEL}\` needs write access to this repository, so it was removed.`, + ); + writeState(null); + return; + } + if (results == null) { + await github.removeLabel(prNumber, APPROVAL_LABEL); + await github.comment( + prNumber, + `There is no visual report for \`${headSha.slice(0, 12)}\` yet, so \`${APPROVAL_LABEL}\` was removed. Apply it again once the report is posted.`, + ); + writeState(null); + return; + } + if (fingerprint != null) { + const key = approvalKey(prNumber, fingerprint); + const record = { + fingerprint, + approver, + approvedAt: new Date().toISOString(), + headSha, + }; + const recordPath = path.join(outDir, 'upload', key); + fs.mkdirSync(path.dirname(recordPath), {recursive: true}); + fs.writeFileSync(recordPath, `${JSON.stringify(record, null, 2)}\n`); + } + } + + writeState({ + prNumber, + headSha, + fingerprint, + labelPresent: event.action === 'labeled', + }); +} + +async function report() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const state = JSON.parse( + fs.readFileSync(path.join(outDir, 'state.json'), 'utf8'), + ); + if (state == null) { + return; + } + const {prNumber, headSha, fingerprint, labelPresent} = state; + const approval = + fingerprint == null + ? null + : await fetchJsonOrNull( + publicUrl( + requireEnv('VISUAL_PUBLIC_BASE_URL'), + approvalKey(prNumber, fingerprint), + ), + ); + const decision = gateDecision({fingerprint, labelPresent, approval}); + await github.createCheckRun({ + name: GATE_CHECK_NAME, + headSha, + conclusion: decision.conclusion, + output: {title: decision.summary, summary: decision.summary}, + }); + console.info(decision.summary); +} + +async function main(command = process.argv[2]) { + if (command === 'prepare') { + await prepare(); + } else if (command === 'report') { + await report(); + } else { + throw new Error('Usage: node approve-run.js prepare|report'); + } +} + +if (require.main === module) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = {latestResults}; diff --git a/.github/workflow-scripts/visual/capture.js b/.github/workflow-scripts/visual/capture.js new file mode 100644 index 000000000000..b19e08dad98b --- /dev/null +++ b/.github/workflow-scripts/visual/capture.js @@ -0,0 +1,205 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Captures every visual flow in one Maestro invocation, so the JVM and + * driver startup (around two minutes on an iOS simulator) is paid once per + * job rather than once per flow. Flows that fail are rerun, and only those, + * on later attempts. + */ + +const childProcess = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const MAESTRO = path.join(process.env.HOME ?? '', '.maestro/bin/maestro'); +const DEFAULT_ATTEMPTS = 3; +const ATTEMPT_TIMEOUT_MS = 30 * 60 * 1000; + +const usage = ` +node capture.js [ios_udid] + +Installs the app, then runs every flow in in one Maestro +invocation, writing screenshots under . Failed flows are retried +up to ${DEFAULT_ATTEMPTS} attempts in total (VISUAL_CAPTURE_ATTEMPTS). +`; + +/** The flows directly inside `flowsDir`, sorted. Helpers live elsewhere. */ +function collectFlows(flowsDir) { + const flows = fs + .readdirSync(flowsDir, {withFileTypes: true}) + .filter(entry => entry.isFile() && /\.ya?ml$/.test(entry.name)) + .map(entry => path.join(flowsDir, entry.name)) + .sort(); + flowsByReportName(flows); + return flows; +} + +/** + * Maps Maestro's report names to flow paths. Maestro reports a flow under its + * file name without the extension, so names must be unique. + */ +function flowsByReportName(flows) { + const byName = new Map(); + for (const flow of flows) { + const name = path.basename(flow).replace(/\.ya?ml$/, ''); + const existing = byName.get(name); + if (existing != null) { + throw new Error( + `${existing} and ${flow} would both report as '${name}'. Rename one.`, + ); + } + byName.set(name, flow); + } + return byName; +} + +/** + * The flows a Maestro JUnit report marks as anything but SUCCESS, in report + * order, matched by the report's file path or, failing that, its name. Throws + * on a report entry that matches none of `flows`. + */ +function failedFlowsFromJUnit(report, flows) { + const byName = flowsByReportName(flows); + const failed = []; + for (const [, attributes] of report.matchAll(/]*)>/g)) { + const name = /\bname="([^"]*)"/.exec(attributes)?.[1]; + const file = /\bfile="([^"]*)"/.exec(attributes)?.[1]; + const status = /\bstatus="([^"]*)"/.exec(attributes)?.[1]; + if (name == null || status === 'SUCCESS') { + continue; + } + const flow = + flows.find(candidate => path.resolve(candidate) === file) ?? + byName.get(name); + if (flow == null) { + throw new Error(`The Maestro report has an unknown flow '${name}'`); + } + failed.push(flow); + } + return failed; +} + +function maestroArgs({flows, appId, outputDir, reportPath, udid, attempt}) { + return [ + ...(udid != null ? [`--udid=${udid}`] : []), + 'test', + ...flows, + '--format=junit', + `--output=${reportPath}`, + '-e', + `APP_ID=${appId}`, + '-e', + `VISUAL_OUTPUT_DIR=${outputDir}`, + // The first attempt installs the driver. Retries reuse it. + ...(attempt > 1 ? ['--no-reinstall-driver'] : []), + ]; +} + +function installApp(platform, appPath, udid) { + if (platform === 'android') { + childProcess.execFileSync('adb', ['install', '-r', appPath], { + stdio: 'inherit', + }); + } else { + childProcess.execFileSync('xcrun', ['simctl', 'install', udid, appPath], { + stdio: 'inherit', + }); + } +} + +function runAttempt(args) { + const result = childProcess.spawnSync(MAESTRO, args, { + stdio: 'inherit', + timeout: ATTEMPT_TIMEOUT_MS, + env: { + ...process.env, + MAESTRO_CLI_NO_ANALYTICS: '1', + MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED: 'true', + MAESTRO_DRIVER_STARTUP_TIMEOUT: '180000', + }, + }); + if (result.error != null) { + throw result.error; + } + return result.status; +} + +function writeSummary(lines) { + const text = `${lines.join('\n')}\n`; + console.info(text); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, text); + } +} + +function main(args = process.argv.slice(2)) { + const [platform, appPath, appId, flowsDir, outputDir, udid] = args; + if ( + !['android', 'ios'].includes(platform) || + outputDir == null || + (platform === 'ios' && udid == null) + ) { + throw new Error(usage); + } + const attempts = Number( + process.env.VISUAL_CAPTURE_ATTEMPTS ?? DEFAULT_ATTEMPTS, + ); + const allFlows = collectFlows(flowsDir); + fs.mkdirSync(outputDir, {recursive: true}); + installApp(platform, appPath, udid); + + const summary = [`### Visual capture (${platform})`, '']; + let pending = allFlows; + for (let attempt = 1; attempt <= attempts && pending.length > 0; attempt++) { + const reportPath = path.resolve(`visual-capture-report-${attempt}.xml`); + const started = Date.now(); + console.info( + `Attempt ${attempt}/${attempts}: ${pending.length} flow(s) in one Maestro run`, + ); + const status = runAttempt( + maestroArgs({ + flows: pending, + appId, + outputDir, + reportPath, + udid, + attempt, + }), + ); + const report = fs.existsSync(reportPath) + ? fs.readFileSync(reportPath, 'utf8') + : ''; + const failed = failedFlowsFromJUnit(report, pending); + if (status !== 0 && failed.length === 0) { + // Maestro failed without attributing it to a flow, for example because + // it could not reach the device. Retrying won't help. + throw new Error(`Maestro exited with ${status} without a failed flow`); + } + summary.push( + `Attempt ${attempt}: ${pending.length - failed.length} of ${pending.length} flow(s) passed in ${Math.round((Date.now() - started) / 1000)} s`, + ); + pending = failed; + } + + if (pending.length > 0) { + summary.push('', `Failed after ${attempts} attempt(s):`); + summary.push(...pending.map(flow => `- \`${flow}\``)); + } + writeSummary(summary); + if (pending.length > 0) { + process.exitCode = 1; + } +} + +if (require.main === module) { + main(); +} + +module.exports = {collectFlows, failedFlowsFromJUnit, maestroArgs}; diff --git a/.github/workflow-scripts/visual/compare-run.js b/.github/workflow-scripts/visual/compare-run.js new file mode 100644 index 000000000000..3a7bb6ca1b7e --- /dev/null +++ b/.github/workflow-scripts/visual/compare-run.js @@ -0,0 +1,441 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Trusted half of the visual regression pipeline. Runs from the default + * branch on `workflow_run`, after the untrusted capture workflow finishes. + * + * node compare-run.js prepare Validate captures, compare with baselines, + * and lay out files to upload under OUT_DIR. + * node compare-run.js report After the upload, create check runs and + * evaluate the approval gate. + */ + +const { + APPROVAL_LABEL, + CHECK_NAME_PREFIX, + GATE_CHECK_NAME, + differenceFingerprint, + gateDecision, +} = require('./approval'); +const {compareCapture, summarize} = require('./compare'); +const {createGitHubClient} = require('./github'); +const { + COMMENT_MARKER, + renderCheckOutput, + renderPullRequestComment, +} = require('./report'); +const { + approvalKey, + baselinePrefix, + fetchBuffer, + fetchJsonOrNull, + findBaseline, + publicUrl, + reportPrefix, +} = require('./storage'); +const {validateCapture} = require('./validate'); +const fs = require('fs'); +const path = require('path'); + +const BASELINE_SEARCH_DEPTH = 20; +// The default Actions token posts as this bot. Only its comments are edited. +const COMMENT_AUTHOR = 'github-actions[bot]'; + +function requireEnv(name) { + const value = process.env[name]; + if (value == null || value === '') { + throw new Error(`${name} is required`); + } + return value; +} + +function writeFile(filePath, contents) { + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + fs.writeFileSync(filePath, contents); +} + +async function resolveContext(github) { + const event = requireEnv('CAPTURE_EVENT'); + const headSha = requireEnv('CAPTURE_HEAD_SHA'); + const headBranch = requireEnv('CAPTURE_HEAD_BRANCH'); + const headRepository = requireEnv('CAPTURE_HEAD_REPOSITORY'); + const defaultBranch = requireEnv('DEFAULT_BRANCH'); + const runId = Number(requireEnv('CAPTURE_RUN_ID')); + const runAttempt = Number(requireEnv('CAPTURE_RUN_ATTEMPT')); + + if (event === 'pull_request') { + const pull = await github.findPullRequest(headSha, headRepository); + if (pull == null) { + return null; + } + return { + kind: 'pull', + prNumber: pull.number, + headSha, + runId, + runAttempt, + defaultBranch, + }; + } + if ( + event === 'push' && + headBranch === defaultBranch && + headRepository === github.repository + ) { + return { + kind: 'main', + prNumber: null, + headSha, + runId, + runAttempt, + defaultBranch, + }; + } + return null; +} + +async function comparePlatform({ + github, + context, + captureDir, + config, + publicBaseUrl, + outDir, + prefix, +}) { + const manifest = validateCapture(captureDir); + const {platform, environmentKey, git} = manifest; + if (git.headSha !== context.headSha) { + throw new Error( + `${platform}: manifest head ${git.headSha} does not match run head ${context.headSha}`, + ); + } + if (!(await github.isOnBranch(git.baseSha, context.defaultBranch))) { + return { + platform, + environmentKey, + manifest, + baseline: null, + results: [], + error: { + title: 'base commit is not on the default branch', + summary: `The capture's base commit \`${git.baseSha}\` is not on \`${context.defaultBranch}\`, so there is no baseline to compare with. Rebase or merge \`${context.defaultBranch}\` and push again.`, + }, + }; + } + + const candidateShas = await github.ancestry( + git.baseSha, + BASELINE_SEARCH_DEPTH, + ); + const baseline = await findBaseline({ + publicBaseUrl, + environmentKey, + candidateShas, + }); + if (baseline == null) { + return { + platform, + environmentKey, + manifest, + baseline: null, + results: [], + error: { + title: 'no baseline for this environment', + summary: `No baseline for environment \`${environmentKey}\` was published for \`${git.baseSha.slice(0, 12)}\` or its ${BASELINE_SEARCH_DEPTH - 1} preceding commits on \`${context.defaultBranch}\`. If the environment key is new, the CI device image changed: the next push to \`${context.defaultBranch}\` publishes a baseline for it. Otherwise check that baseline publication on \`${context.defaultBranch}\` is succeeding.`, + }, + }; + } + + const expectedUrl = id => + publicUrl( + publicBaseUrl, + `${baselinePrefix(environmentKey, baseline.sha)}/${id}.png`, + ); + const results = await compareCapture({ + manifest, + baselineManifest: baseline.manifest, + readExpected: id => fetchBuffer(expectedUrl(id)), + readCaptured: id => + fs.promises.readFile(path.join(captureDir, `${id}.png`)), + config, + }); + + for (const result of results) { + if (result.status === 'unchanged') { + continue; + } + const platformPrefix = `${prefix}/${platform}`; + if (result.captured != null) { + writeFile( + path.join( + outDir, + 'upload', + platformPrefix, + 'captured', + `${result.id}.png`, + ), + fs.readFileSync(path.join(captureDir, `${result.id}.png`)), + ); + } + if (result.diffImage != null) { + writeFile( + path.join(outDir, 'upload', platformPrefix, 'diff', `${result.id}.png`), + result.diffImage, + ); + } + } + + return { + platform, + environmentKey, + manifest, + baseline: {sha: baseline.sha, distance: baseline.distance}, + results, + error: null, + }; +} + +function stageBaseline({outDir, captureDir, manifest, sha}) { + const prefix = baselinePrefix(manifest.environmentKey, sha); + for (const image of manifest.images) { + writeFile( + path.join(outDir, 'baseline-images', prefix, `${image.id}.png`), + fs.readFileSync(path.join(captureDir, `${image.id}.png`)), + ); + } + // Uploaded after the images: a baseline exists once its manifest does. + writeFile( + path.join(outDir, 'baseline-manifests', prefix, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + ); +} + +function serializableResults(results) { + return results.map(({diffImage, ...result}) => ({ + ...result, + hasDiffImage: diffImage != null, + })); +} + +async function prepare() { + const github = createGitHubClient(); + const artifactsDir = requireEnv('VISUAL_ARTIFACTS_DIR'); + const outDir = requireEnv('OUT_DIR'); + const publicBaseUrl = requireEnv('VISUAL_PUBLIC_BASE_URL'); + const config = JSON.parse( + fs.readFileSync(requireEnv('VISUAL_CONFIG'), 'utf8'), + ); + + const context = await resolveContext(github); + if (context == null) { + console.info('Not an open pull request or a push to the default branch.'); + writeFile(path.join(outDir, 'state.json'), 'null\n'); + return; + } + const prefix = reportPrefix(context); + + const platforms = []; + for (const entry of fs.readdirSync(artifactsDir, {withFileTypes: true})) { + if (!entry.isDirectory() || !entry.name.startsWith('visual-capture-')) { + continue; + } + const captureDir = path.join(artifactsDir, entry.name); + const result = await comparePlatform({ + github, + context, + captureDir, + config, + publicBaseUrl, + outDir, + prefix, + }); + if (context.kind === 'main') { + stageBaseline({ + outDir, + captureDir, + manifest: result.manifest, + sha: context.headSha, + }); + } + platforms.push(result); + } + + const fingerprint = differenceFingerprint( + platforms.map(platform => ({ + platform: platform.platform, + environmentKey: platform.environmentKey, + baselineSha: platform.baseline?.sha, + results: platform.results, + })), + ); + const state = { + context, + reportPrefix: prefix, + fingerprint, + platforms: platforms.map(({manifest, results, ...platform}) => ({ + ...platform, + git: manifest.git, + results: serializableResults(results), + })), + }; + writeFile( + path.join(outDir, 'upload', prefix, 'results.json'), + `${JSON.stringify(state, null, 2)}\n`, + ); + writeFile(path.join(outDir, 'state.json'), `${JSON.stringify(state)}\n`); + + for (const platform of state.platforms) { + const {counts} = summarize(platform.results); + console.info( + `${platform.platform} (${platform.environmentKey}): ${ + platform.error?.title ?? JSON.stringify(counts) + }`, + ); + } +} + +async function report() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const publicBaseUrl = requireEnv('VISUAL_PUBLIC_BASE_URL'); + const state = JSON.parse( + fs.readFileSync(path.join(outDir, 'state.json'), 'utf8'), + ); + if (state == null) { + return; + } + const {context, reportPrefix: prefix, fingerprint} = state; + const detailsUrl = publicUrl(publicBaseUrl, `${prefix}/results.json`); + + const summaryLines = []; + const commentPlatforms = []; + for (const platform of state.platforms) { + const imageUrl = (kind, id) => + kind === 'expected' + ? publicUrl( + publicBaseUrl, + `${baselinePrefix(platform.environmentKey, platform.baseline.sha)}/${id}.png`, + ) + : publicUrl( + publicBaseUrl, + `${prefix}/${platform.platform}/${kind}/${id}.png`, + ); + const output = renderCheckOutput({...platform, imageUrl}); + const {hasDifferences} = summarize(platform.results); + const checkRun = await github.createCheckRun({ + name: `${CHECK_NAME_PREFIX} (${platform.platform})`, + headSha: context.headSha, + conclusion: + platform.error != null + ? 'neutral' + : hasDifferences + ? 'failure' + : 'success', + output, + detailsUrl, + }); + commentPlatforms.push({...platform, imageUrl, checkUrl: checkRun.html_url}); + summaryLines.push( + `## ${output.title}`, + '', + output.summary, + '', + output.text, + '', + ); + } + + if (context.kind === 'pull') { + const labels = await github.labels(context.prNumber); + const approval = + fingerprint == null + ? null + : await fetchJsonOrNull( + publicUrl( + publicBaseUrl, + approvalKey(context.prNumber, fingerprint), + ), + ); + const decision = gateDecision({ + fingerprint, + labelPresent: labels.includes(APPROVAL_LABEL), + approval, + }); + if (decision.staleLabel) { + await github.removeLabel(context.prNumber, APPROVAL_LABEL); + await github.comment( + context.prNumber, + `The visual differences changed in \`${context.headSha.slice(0, 12)}\`, so \`${APPROVAL_LABEL}\` was removed. Review the new report and apply it again to approve.`, + ); + } + await github.createCheckRun({ + name: GATE_CHECK_NAME, + headSha: context.headSha, + conclusion: decision.conclusion, + output: {title: decision.summary, summary: decision.summary}, + detailsUrl, + }); + summaryLines.push(`## Approval`, '', decision.summary, ''); + await upsertComment({ + github, + prNumber: context.prNumber, + body: renderPullRequestComment({ + platforms: commentPlatforms, + headSha: context.headSha, + gateSummary: decision.summary, + }), + hasContent: + fingerprint != null || + commentPlatforms.some(platform => platform.error != null), + }); + } + + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summaryLines.join('\n')); + } +} + +/** + * Keeps one report comment per PR up to date. A PR that never had + * differences gets no comment. Once it has one, later runs edit it, including + * to say the differences are gone. + */ +async function upsertComment({github, prNumber, body, hasContent}) { + const existing = await github.findComment( + prNumber, + COMMENT_MARKER, + COMMENT_AUTHOR, + ); + if (existing != null) { + await github.updateComment(existing.id, body); + } else if (hasContent) { + await github.comment(prNumber, body); + } +} + +async function main(command = process.argv[2]) { + if (command === 'prepare') { + await prepare(); + } else if (command === 'report') { + await report(); + } else { + throw new Error('Usage: node compare-run.js prepare|report'); + } +} + +if (require.main === module) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = {resolveContext}; diff --git a/.github/workflow-scripts/visual/compare.js b/.github/workflow-scripts/visual/compare.js new file mode 100644 index 000000000000..86694966dc4b --- /dev/null +++ b/.github/workflow-scripts/visual/compare.js @@ -0,0 +1,152 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const pixelmatch = require('pixelmatch'); +const {PNG} = require('pngjs'); + +// Strict by default: any changed pixel fails. Loosen per test in +// packages/rn-tester/visual-tests/config.json, with evidence. +const DEFAULT_POLICY = { + // pixelmatch's per-pixel color distance threshold, from 0 to 1. + threshold: 0, + // Count anti-aliased pixels as differences. + includeAA: true, + maxDiffPixels: 0, + // Optional: also fail when this share of the image changes. + maxDiffRatio: null, +}; + +function policyFor(testId, config) { + return {...DEFAULT_POLICY, ...config?.defaults, ...config?.tests?.[testId]}; +} + +/** + * Compares two PNG buffers. Returns the outcome and, when the images have + * the same dimensions and differ, a diff image highlighting changed pixels. + */ +function compareImages(expectedBuffer, capturedBuffer, policy) { + const expected = PNG.sync.read(expectedBuffer); + const captured = PNG.sync.read(capturedBuffer); + if ( + expected.width !== captured.width || + expected.height !== captured.height + ) { + return { + status: 'dimension-mismatch', + expectedSize: {width: expected.width, height: expected.height}, + capturedSize: {width: captured.width, height: captured.height}, + diffPixels: null, + diffRatio: null, + diff: null, + }; + } + + const {width, height} = expected; + const diff = new PNG({width, height}); + const diffPixels = pixelmatch( + expected.data, + captured.data, + diff.data, + width, + height, + {threshold: policy.threshold, includeAA: policy.includeAA}, + ); + const diffRatio = diffPixels / (width * height); + const changed = + diffPixels > policy.maxDiffPixels || + (policy.maxDiffRatio != null && diffRatio > policy.maxDiffRatio); + return { + status: changed ? 'changed' : 'unchanged', + diffPixels, + diffRatio, + diff: diffPixels > 0 ? PNG.sync.write(diff) : null, + }; +} + +/** + * Compares every captured image with the baseline. `readExpected(id)` and + * `readCaptured(id)` return PNG buffers. Results are sorted by test ID. + */ +async function compareCapture({ + manifest, + baselineManifest, + readExpected, + readCaptured, + config, +}) { + const expectedById = new Map( + (baselineManifest?.images ?? []).map(image => [image.id, image]), + ); + const results = []; + + for (const image of manifest.images) { + const expected = expectedById.get(image.id); + expectedById.delete(image.id); + if (expected == null) { + results.push({id: image.id, status: 'added', captured: image}); + continue; + } + if (expected.sha256 === image.sha256) { + results.push({ + id: image.id, + status: 'unchanged', + diffPixels: 0, + diffRatio: 0, + captured: image, + expected, + }); + continue; + } + const policy = policyFor(image.id, config); + const {diff, ...outcome} = compareImages( + await readExpected(image.id), + await readCaptured(image.id), + policy, + ); + results.push({ + id: image.id, + ...outcome, + policy, + captured: image, + expected, + diffImage: diff, + }); + } + + for (const expected of expectedById.values()) { + results.push({id: expected.id, status: 'removed', expected}); + } + + return results.sort((a, b) => a.id.localeCompare(b.id)); +} + +function summarize(results) { + const counts = { + unchanged: 0, + changed: 0, + 'dimension-mismatch': 0, + added: 0, + removed: 0, + }; + for (const result of results) { + counts[result.status] += 1; + } + return { + counts, + hasDifferences: results.some(result => result.status !== 'unchanged'), + }; +} + +module.exports = { + DEFAULT_POLICY, + compareCapture, + compareImages, + policyFor, + summarize, +}; diff --git a/.github/workflow-scripts/visual/github.js b/.github/workflow-scripts/visual/github.js new file mode 100644 index 000000000000..9dd1645207f9 --- /dev/null +++ b/.github/workflow-scripts/visual/github.js @@ -0,0 +1,145 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +function createGitHubClient({ + token = process.env.GITHUB_TOKEN, + repository = process.env.GITHUB_REPOSITORY, + apiUrl = process.env.GITHUB_API_URL ?? 'https://api.github.com', +} = {}) { + if (!token || !repository) { + throw new Error('GITHUB_TOKEN and GITHUB_REPOSITORY are required'); + } + + async function request(method, route, body) { + const response = await fetch(`${apiUrl}/repos/${repository}${route}`, { + method, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + ...(body != null ? {'Content-Type': 'application/json'} : null), + }, + body: body != null ? JSON.stringify(body) : undefined, + }); + if (response.status === 404 && method === 'GET') { + return null; + } + if (!response.ok) { + throw new Error( + `${method} ${route} failed with ${response.status}: ${await response.text()}`, + ); + } + return response.status === 204 ? null : response.json(); + } + + return { + repository, + + /** Finds the open PR whose head is `headSha` in `headRepository`. */ + async findPullRequest(headSha, headRepository) { + for (let page = 1; page <= 10; page++) { + const pulls = await request( + 'GET', + `/pulls?state=open&sort=updated&direction=desc&per_page=100&page=${page}`, + ); + const match = pulls.find( + pull => + pull.head.sha === headSha && + pull.head.repo?.full_name === headRepository, + ); + if (match != null || pulls.length < 100) { + return match ?? null; + } + } + return null; + }, + + /** Whether `sha` is an ancestor of, or equal to, `branch`. */ + async isOnBranch(sha, branch) { + const comparison = await request( + 'GET', + `/compare/${sha}...${encodeURIComponent(branch)}`, + ); + return ( + comparison != null && + (comparison.status === 'ahead' || comparison.status === 'identical') + ); + }, + + /** `sha` and up to `count - 1` of its ancestors, newest first. */ + async ancestry(sha, count) { + const commits = await request( + 'GET', + `/commits?sha=${sha}&per_page=${count}`, + ); + return (commits ?? []).map(commit => commit.sha); + }, + + async labels(prNumber) { + const labels = await request( + 'GET', + `/issues/${prNumber}/labels?per_page=100`, + ); + return (labels ?? []).map(label => label.name); + }, + + removeLabel(prNumber, label) { + return request( + 'DELETE', + `/issues/${prNumber}/labels/${encodeURIComponent(label)}`, + ); + }, + + comment(prNumber, body) { + return request('POST', `/issues/${prNumber}/comments`, {body}); + }, + + /** The first comment by `author` on the PR that contains `marker`. */ + async findComment(prNumber, marker, author) { + for (let page = 1; ; page++) { + const comments = await request( + 'GET', + `/issues/${prNumber}/comments?per_page=100&page=${page}`, + ); + const match = (comments ?? []).find( + comment => + comment.user?.login === author && comment.body?.includes(marker), + ); + if (match != null || (comments ?? []).length < 100) { + return match ?? null; + } + } + }, + + updateComment(commentId, body) { + return request('PATCH', `/issues/comments/${commentId}`, {body}); + }, + + async permission(login) { + const result = await request( + 'GET', + `/collaborators/${encodeURIComponent(login)}/permission`, + ); + return result?.permission ?? 'none'; + }, + + createCheckRun({name, headSha, conclusion, output, detailsUrl}) { + return request('POST', '/check-runs', { + name, + head_sha: headSha, + status: 'completed', + conclusion, + output, + details_url: detailsUrl, + }); + }, + }; +} + +module.exports = {createGitHubClient}; diff --git a/.github/workflow-scripts/visual/manifest.js b/.github/workflow-scripts/visual/manifest.js new file mode 100644 index 000000000000..5bbcfbcaf3f8 --- /dev/null +++ b/.github/workflow-scripts/visual/manifest.js @@ -0,0 +1,267 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {readPngDimensions} = require('./png'); +const childProcess = require('child_process'); +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const MANIFEST_VERSION = 1; +const PLATFORMS = ['android', 'ios']; + +// //, each segment lowercase kebab-case. +const TEST_ID_PATTERN = + /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*){2}$/; +const SHA_PATTERN = /^[0-9a-f]{40}$/; + +// Environment properties that select a baseline. Anything else in the +// environment is recorded for diagnosis but does not change the key. +const KEYED_PROPERTIES = { + android: [ + 'apiLevel', + 'abi', + 'nativeBridge', + 'systemImage', + 'screenSize', + 'density', + 'fontScale', + 'locale', + 'animatorDurationScale', + ], + ios: ['runtime', 'deviceType', 'appearance', 'contentSizeCategory', 'locale'], +}; + +function sha256(buffer) { + return crypto.createHash('sha256').update(buffer).digest('hex'); +} + +function canonicalJson(value) { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (value != null && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +function slug(value) { + return String(value) + .toLowerCase() + .replace(/[^a-z0-9.]+/g, '-') + .replace(/^-|-$/g, ''); +} + +/** + * A readable prefix for the properties people look for first, followed by a + * hash of every keyed property so that drift in any of them selects a + * different baseline instead of producing a spurious diff. + */ +function environmentKey(platform, environment) { + const keyed = {}; + for (const property of KEYED_PROPERTIES[platform]) { + keyed[property] = environment[property] ?? null; + } + const hash = sha256(Buffer.from(canonicalJson({platform, keyed}))).slice( + 0, + 8, + ); + const readable = + platform === 'android' + ? [ + 'android', + `api${environment.apiLevel}`, + environment.abi, + environment.screenSize, + `${environment.density}dpi`, + ] + : [ + 'ios', + String(environment.runtime).replace(/^iOS-/, ''), + environment.deviceType, + ]; + return [...readable.map(slug), hash].join('-'); +} + +function collectImages(captureDir) { + const images = []; + const visit = directory => { + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(entryPath); + } else if (entry.isFile() && entry.name.endsWith('.png')) { + images.push(entryPath); + } + } + }; + visit(captureDir); + return images.sort(); +} + +function buildManifest({captureDir, platform, environment, git}) { + const images = collectImages(captureDir).map(imagePath => { + const relativePath = path + .relative(captureDir, imagePath) + .split(path.sep) + .join('/'); + const buffer = fs.readFileSync(imagePath); + const dimensions = readPngDimensions(buffer); + if (dimensions == null) { + throw new Error(`Not a PNG: ${relativePath}`); + } + return { + id: relativePath.replace(/\.png$/, ''), + path: relativePath, + ...dimensions, + bytes: buffer.length, + sha256: sha256(buffer), + }; + }); + + return { + version: MANIFEST_VERSION, + platform, + environmentKey: environmentKey(platform, environment), + environment, + git, + images, + }; +} + +function run(command, args) { + return childProcess + .execFileSync(command, args, {encoding: 'utf8', timeout: 30000}) + .trim(); +} + +function readAndroidEnvironment() { + const getprop = name => run('adb', ['shell', 'getprop', name]); + const setting = (namespace, name) => + run('adb', ['shell', 'settings', 'get', namespace, name]); + const wm = name => { + const output = run('adb', ['shell', 'wm', name]); + const override = /Override \w+: (\S+)/.exec(output); + return (override ?? /Physical \w+: (\S+)/.exec(output))?.[1] ?? output; + }; + return { + apiLevel: Number(getprop('ro.build.version.sdk')), + abi: getprop('ro.product.cpu.abi'), + nativeBridge: getprop('ro.dalvik.vm.native.bridge'), + systemImage: getprop('ro.build.fingerprint'), + screenSize: wm('size'), + density: Number(wm('density')), + fontScale: setting('system', 'font_scale'), + locale: getprop('persist.sys.locale') || getprop('ro.product.locale'), + animatorDurationScale: setting('global', 'animator_duration_scale'), + }; +} + +function readIosEnvironment(udid) { + const devices = JSON.parse( + run('xcrun', ['simctl', 'list', 'devices', '-j']), + ).devices; + let device = null; + let runtime = null; + for (const [runtimeId, runtimeDevices] of Object.entries(devices)) { + const match = runtimeDevices.find(candidate => candidate.udid === udid); + if (match != null) { + device = match; + runtime = runtimeId.replace('com.apple.CoreSimulator.SimRuntime.', ''); + } + } + if (device == null) { + throw new Error(`Simulator ${udid} not found`); + } + const ui = name => run('xcrun', ['simctl', 'ui', udid, name]); + return { + runtime, + deviceType: device.deviceTypeIdentifier.replace( + 'com.apple.CoreSimulator.SimDeviceType.', + '', + ), + appearance: ui('appearance'), + contentSizeCategory: ui('content_size'), + locale: run('xcrun', [ + 'simctl', + 'spawn', + udid, + 'defaults', + 'read', + '-g', + 'AppleLocale', + ]), + xcode: run('xcodebuild', ['-version']).split('\n').join(' '), + }; +} + +function readGitInfo() { + const git = args => run('git', args); + return { + // For pull_request events actions/checkout tests the merge commit, whose + // first parent is the base branch commit the PR was tested against. + testedSha: git(['rev-parse', 'HEAD']), + baseSha: git(['rev-parse', 'HEAD^1']), + headSha: process.env.VISUAL_HEAD_SHA || git(['rev-parse', 'HEAD']), + event: process.env.GITHUB_EVENT_NAME ?? null, + ref: process.env.GITHUB_REF ?? null, + }; +} + +const usage = ` +node manifest.js [ios_udid] + +Writes a manifest for the PNGs under , reading the environment +from the running emulator (adb) or the simulator with . +`; + +function main(args = process.argv.slice(2)) { + const [platform, captureDir, outputPath, udid] = args; + if ( + !PLATFORMS.includes(platform) || + captureDir == null || + outputPath == null + ) { + throw new Error(usage); + } + const environment = + platform === 'android' + ? readAndroidEnvironment() + : readIosEnvironment(udid); + const manifest = buildManifest({ + captureDir, + platform, + environment, + git: readGitInfo(), + }); + fs.writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`); + console.info( + `Wrote ${manifest.images.length} image(s) for ${manifest.environmentKey} to ${outputPath}`, + ); +} + +if (require.main === module) { + main(); +} + +module.exports = { + KEYED_PROPERTIES, + MANIFEST_VERSION, + PLATFORMS, + SHA_PATTERN, + TEST_ID_PATTERN, + buildManifest, + canonicalJson, + environmentKey, + sha256, +}; diff --git a/.github/workflow-scripts/visual/png.js b/.github/workflow-scripts/visual/png.js new file mode 100644 index 000000000000..73d9e5df978b --- /dev/null +++ b/.github/workflow-scripts/visual/png.js @@ -0,0 +1,30 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const PNG_SIGNATURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); + +/** + * Reads width and height from a PNG's IHDR chunk without decoding pixels, so + * that untrusted files can be checked before any image library sees them. + * Returns null if the buffer does not start with a well-formed PNG header. + */ +function readPngDimensions(buffer) { + if ( + buffer.length < 24 || + !buffer.subarray(0, 8).equals(PNG_SIGNATURE) || + buffer.toString('ascii', 12, 16) !== 'IHDR' + ) { + return null; + } + return {width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20)}; +} + +module.exports = {readPngDimensions}; diff --git a/.github/workflow-scripts/visual/report.js b/.github/workflow-scripts/visual/report.js new file mode 100644 index 000000000000..7063847c2add --- /dev/null +++ b/.github/workflow-scripts/visual/report.js @@ -0,0 +1,217 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {summarize} = require('./compare'); + +// GitHub limits check run output text to 65535 characters and 50 images. +const MAX_TEXT_LENGTH = 60000; +const MAX_IMAGES = 50; + +const STATUS_LABELS = { + changed: 'Changed', + 'dimension-mismatch': 'Size changed', + added: 'New', + removed: 'Removed', + unchanged: 'Unchanged', +}; + +// GitHub limits comment bodies to 65536 characters. Each changed test costs +// about 1 KB of Markdown, so later tests are listed by name only. +const MAX_COMMENT_TESTS = 20; + +const COMMENT_MARKER = ''; + +function formatPercent(ratio) { + return ratio == null ? '' : `${(ratio * 100).toFixed(3)}%`; +} + +/** How a differing result changed, for example "9633 px (17.502%)". */ +function describeChange(result) { + if (result.status === 'dimension-mismatch') { + return `${result.expectedSize.width}×${result.expectedSize.height} → ${result.capturedSize.width}×${result.capturedSize.height}`; + } + return result.diffPixels != null + ? `${result.diffPixels} px (${formatPercent(result.diffRatio)})` + : ''; +} + +function heading(result) { + const change = describeChange(result); + return `\`${result.id}\`: ${STATUS_LABELS[result.status]}${change ? `, ${change}` : ''}`; +} + +/** + * Renders one platform's comparison as GitHub check run output. `imageUrl` + * maps (kind, test ID) to the public URL of an uploaded expected, captured + * or diff image. + */ +function renderCheckOutput({ + platform, + environmentKey, + baseline, + results, + imageUrl, + error, +}) { + if (error != null) { + return { + title: `${platform}: ${error.title}`, + summary: error.summary, + text: '', + images: [], + }; + } + + const {counts, hasDifferences} = summarize(results); + const differing = results.filter(result => result.status !== 'unchanged'); + const baselineLine = + baseline.distance === 0 + ? `Compared with the baseline for base commit \`${baseline.sha.slice(0, 12)}\`.` + : `No baseline was published for the base commit, so this compares with the nearest ancestor that has one, \`${baseline.sha.slice(0, 12)}\` (${baseline.distance} commit(s) earlier). Differences can come from commits in between.`; + + const summary = [ + hasDifferences + ? `${differing.length} of ${results.length} visual test(s) differ.` + : `All ${results.length} visual test(s) match.`, + '', + baselineLine, + '', + `Environment: \`${environmentKey}\``, + '', + Object.entries(counts) + .filter(([, count]) => count > 0) + .map(([status, count]) => `${STATUS_LABELS[status]}: ${count}`) + .join(' · '), + ].join('\n'); + + const sections = []; + const images = []; + for (const result of differing) { + const row = []; + const cell = kind => ``; + if (result.expected != null) { + row.push(['Expected', cell('expected')]); + } + if (result.captured != null) { + row.push(['Captured', cell('captured')]); + } + if (result.hasDiffImage) { + row.push(['Diff', cell('diff')]); + } + const details = describeChange(result); + sections.push( + [ + `### ${heading(result)}`, + '', + `| ${row.map(([label]) => label).join(' | ')} |`, + `|${row.map(() => '---').join('|')}|`, + `| ${row.map(([, html]) => html).join(' | ')} |`, + ].join('\n'), + ); + if (result.hasDiffImage && images.length < MAX_IMAGES) { + images.push({ + alt: `${result.id} diff`, + image_url: imageUrl('diff', result.id), + caption: `${result.id}: ${details}`, + }); + } + } + + let text = sections.join('\n\n'); + if (text.length > MAX_TEXT_LENGTH) { + text = `${text.slice(0, MAX_TEXT_LENGTH)}\n\n…truncated. See results.json for the full list.`; + } + + return { + title: hasDifferences + ? `${platform}: ${differing.length} visual difference(s)` + : `${platform}: no visual differences`, + summary, + text, + images, + }; +} + +/** + * Renders the pull request comment for all platforms. Each changed test + * shows its diff, with the expected and captured images in a collapsed + * section. `platforms` carry the same fields as `renderCheckOutput` takes, + * plus `checkUrl`, the platform's check run. + */ +function renderPullRequestComment({platforms, headSha, gateSummary}) { + const differing = platforms.flatMap(platform => + platform.error != null + ? [] + : platform.results + .filter(result => result.status !== 'unchanged') + .map(result => ({platform, result})), + ); + const lines = [COMMENT_MARKER, '## Visual regression', '']; + lines.push( + differing.length > 0 + ? `**${differing.length} visual difference(s)** at \`${headSha.slice(0, 12)}\`. ${gateSummary}` + : `No visual differences at \`${headSha.slice(0, 12)}\`.`, + '', + ); + for (const platform of platforms) { + if (platform.error != null) { + lines.push( + `**${platform.platform}:** [${platform.error.title}](${platform.checkUrl})`, + '', + ); + } + } + + for (const [index, {platform, result}] of differing.entries()) { + if (index === MAX_COMMENT_TESTS) { + lines.push( + `${differing.length - index} more:`, + '', + ...differing + .slice(index) + .map( + ({platform: more, result: rest}) => + `- ${more.platform} ${heading(rest)}, [check run](${more.checkUrl})`, + ), + '', + ); + break; + } + const image = (kind, width) => + `${kind}`; + lines.push(`### ${platform.platform} ${heading(result)}`, ''); + if (result.hasDiffImage) { + lines.push(image('diff', 360), ''); + } + const panels = []; + if (result.expected != null) { + panels.push(['Expected', image('expected', 250)]); + } + if (result.captured != null) { + panels.push(['Captured', image('captured', 250)]); + } + if (panels.length > 0) { + lines.push( + `${panels + .map(([label]) => label) + .join(' and ')}`, + '', + `| ${panels.map(([label]) => label).join(' | ')} |`, + `|${panels.map(() => '---').join('|')}|`, + `| ${panels.map(([, html]) => html).join(' | ')} |`, + '', + '
', + '', + ); + } + } + return lines.join('\n'); +} + +module.exports = {COMMENT_MARKER, renderCheckOutput, renderPullRequestComment}; diff --git a/.github/workflow-scripts/visual/storage.js b/.github/workflow-scripts/visual/storage.js new file mode 100644 index 000000000000..e84fd76e5da0 --- /dev/null +++ b/.github/workflow-scripts/visual/storage.js @@ -0,0 +1,140 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {SHA_PATTERN} = require('./manifest'); + +// Object layout. Every key is write-once: reports and approvals include the +// run that produced them, and a baseline is only ever written for a new SHA. +// +// baselines///manifest.json +// baselines///.png +// reports/pull///-/results.json +// reports/pull///-//{expected,captured,diff}/.png +// reports/main//-/... +// approvals/pull//.json + +function assertSha(sha) { + if (!SHA_PATTERN.test(sha)) { + throw new Error(`Invalid SHA: ${String(sha)}`); + } +} + +function assertPositiveInteger(value, name) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`Invalid ${name}: ${String(value)}`); + } +} + +function baselinePrefix(environmentKey, sha) { + assertSha(sha); + return `baselines/${environmentKey}/${sha}`; +} + +function reportPrefix({prNumber, headSha, runId, runAttempt}) { + assertSha(headSha); + assertPositiveInteger(runId, 'run ID'); + assertPositiveInteger(runAttempt, 'run attempt'); + if (prNumber == null) { + return `reports/main/${headSha}/${runId}-${runAttempt}`; + } + assertPositiveInteger(prNumber, 'PR number'); + return `reports/pull/${prNumber}/${headSha}/${runId}-${runAttempt}`; +} + +function approvalKey(prNumber, fingerprint) { + assertPositiveInteger(prNumber, 'PR number'); + if (!/^[0-9a-f]{16}$/.test(fingerprint)) { + throw new Error(`Invalid fingerprint: ${String(fingerprint)}`); + } + return `approvals/pull/${prNumber}/${fingerprint}.json`; +} + +function publicUrl(publicBaseUrl, key) { + return `${publicBaseUrl.replace(/\/$/, '')}/${key + .split('/') + .map(encodeURIComponent) + .join('/')}`; +} + +async function fetchOrNull(url) { + const response = await fetch(url, {cache: 'no-store'}); + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`GET ${url} failed with ${response.status}`); + } + return response; +} + +async function fetchJsonOrNull(url) { + const response = await fetchOrNull(url); + return response == null ? null : response.json(); +} + +async function fetchBuffer(url) { + const response = await fetchOrNull(url); + if (response == null) { + throw new Error(`GET ${url} returned 404`); + } + return Buffer.from(await response.arrayBuffer()); +} + +/** + * Lists the immediate "subdirectories" under a prefix, using the public GCS + * JSON API. + */ +async function listPrefixes(bucket, prefix) { + const prefixes = []; + let pageToken = null; + do { + const url = new URL( + `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(bucket)}/o`, + ); + url.searchParams.set('prefix', prefix); + url.searchParams.set('delimiter', '/'); + if (pageToken != null) { + url.searchParams.set('pageToken', pageToken); + } + const page = await fetchJsonOrNull(url.toString()); + prefixes.push(...(page?.prefixes ?? [])); + pageToken = page?.nextPageToken ?? null; + } while (pageToken != null); + return prefixes; +} + +/** + * Returns the first of `candidateShas` (newest first) that has a published + * baseline for `environmentKey`, with its manifest, or null. + */ +async function findBaseline({publicBaseUrl, environmentKey, candidateShas}) { + for (const [distance, sha] of candidateShas.entries()) { + const manifest = await fetchJsonOrNull( + publicUrl( + publicBaseUrl, + `${baselinePrefix(environmentKey, sha)}/manifest.json`, + ), + ); + if (manifest != null) { + return {sha, distance, manifest}; + } + } + return null; +} + +module.exports = { + approvalKey, + baselinePrefix, + fetchBuffer, + fetchJsonOrNull, + findBaseline, + listPrefixes, + publicUrl, + reportPrefix, +}; diff --git a/.github/workflow-scripts/visual/validate.js b/.github/workflow-scripts/visual/validate.js new file mode 100644 index 000000000000..0b6361d420e5 --- /dev/null +++ b/.github/workflow-scripts/visual/validate.js @@ -0,0 +1,183 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + MANIFEST_VERSION, + PLATFORMS, + SHA_PATTERN, + TEST_ID_PATTERN, + sha256, +} = require('./manifest'); +const {readPngDimensions} = require('./png'); +const fs = require('fs'); +const path = require('path'); + +const LIMITS = { + maxImages: 200, + maxImageBytes: 5 * 1024 * 1024, + maxTotalBytes: 100 * 1024 * 1024, + maxDimension: 4096, + maxManifestBytes: 1024 * 1024, +}; + +const ENVIRONMENT_KEY_PATTERN = /^(android|ios)-[a-z0-9.-]{1,120}-[0-9a-f]{8}$/; + +/** + * Validates a capture artifact produced by untrusted pull request code. The + * artifact is data only: nothing in it is executed, and every path is checked + * before it is read. Returns the parsed manifest, or throws an Error listing + * every problem found. + */ +function validateCapture(captureDir, limits = LIMITS) { + const errors = []; + const manifestPath = path.join(captureDir, 'manifest.json'); + const manifestStat = lstatOrNull(manifestPath); + if (manifestStat == null || !manifestStat.isFile()) { + throw new Error('manifest.json is missing or not a regular file'); + } + if (manifestStat.size > limits.maxManifestBytes) { + throw new Error( + `manifest.json is larger than ${limits.maxManifestBytes} bytes`, + ); + } + + let manifest; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch (error) { + throw new Error(`manifest.json is not valid JSON: ${error.message}`); + } + + if (manifest?.version !== MANIFEST_VERSION) { + errors.push(`Unsupported manifest version: ${String(manifest?.version)}`); + } + if (!PLATFORMS.includes(manifest?.platform)) { + errors.push(`Unknown platform: ${String(manifest?.platform)}`); + } + if ( + typeof manifest?.environmentKey !== 'string' || + !ENVIRONMENT_KEY_PATTERN.test(manifest.environmentKey) || + !manifest.environmentKey.startsWith(`${manifest.platform}-`) + ) { + errors.push(`Invalid environment key: ${String(manifest?.environmentKey)}`); + } + for (const field of ['testedSha', 'baseSha', 'headSha']) { + if (!SHA_PATTERN.test(String(manifest?.git?.[field]))) { + errors.push(`Invalid git.${field}`); + } + } + + const images = Array.isArray(manifest?.images) ? manifest.images : null; + if (images == null) { + errors.push('images must be an array'); + } else if (images.length > limits.maxImages) { + errors.push(`Too many images: ${images.length} > ${limits.maxImages}`); + } else { + const seen = new Set(); + let totalBytes = 0; + for (const image of images) { + const id = String(image?.id); + if (!TEST_ID_PATTERN.test(id)) { + errors.push(`Invalid test ID: ${JSON.stringify(image?.id)}`); + continue; + } + if (seen.has(id)) { + errors.push(`Duplicate test ID: ${id}`); + continue; + } + seen.add(id); + // The path is derived from the validated ID, never taken from the + // manifest, so it cannot escape the capture directory. + if (image.path !== `${id}.png`) { + errors.push(`${id}: path must be ${id}.png`); + continue; + } + const imagePath = path.join(captureDir, ...id.split('/')) + '.png'; + const stat = lstatOrNull(imagePath); + if (stat == null || !stat.isFile()) { + errors.push(`${id}: file is missing or not a regular file`); + continue; + } + if (stat.size > limits.maxImageBytes) { + errors.push( + `${id}: ${stat.size} bytes exceeds ${limits.maxImageBytes}`, + ); + continue; + } + totalBytes += stat.size; + const buffer = fs.readFileSync(imagePath); + const dimensions = readPngDimensions(buffer); + if (dimensions == null) { + errors.push(`${id}: not a PNG`); + continue; + } + if ( + dimensions.width === 0 || + dimensions.height === 0 || + dimensions.width > limits.maxDimension || + dimensions.height > limits.maxDimension + ) { + errors.push( + `${id}: ${dimensions.width}x${dimensions.height} is outside 1-${limits.maxDimension} px`, + ); + continue; + } + if ( + image.width !== dimensions.width || + image.height !== dimensions.height || + image.bytes !== stat.size || + image.sha256 !== sha256(buffer) + ) { + errors.push(`${id}: file does not match its manifest entry`); + } + } + if (totalBytes > limits.maxTotalBytes) { + errors.push(`Total size ${totalBytes} exceeds ${limits.maxTotalBytes}`); + } + const unlisted = listFiles(captureDir).filter( + file => file !== 'manifest.json' && !seen.has(file.replace(/\.png$/, '')), + ); + if (unlisted.length > 0) { + errors.push( + `Files not in the manifest: ${unlisted.slice(0, 10).join(', ')}`, + ); + } + } + + if (errors.length > 0) { + throw new Error(`Invalid capture artifact:\n- ${errors.join('\n- ')}`); + } + return manifest; +} + +function lstatOrNull(filePath) { + try { + return fs.lstatSync(filePath); + } catch { + return null; + } +} + +function listFiles(root) { + const files = []; + const visit = (directory, prefix) => { + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { + const relative = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) { + visit(path.join(directory, entry.name), relative); + } else { + files.push(relative); + } + } + }; + visit(root, ''); + return files; +} + +module.exports = {LIMITS, validateCapture}; diff --git a/.github/workflows/visual-approve-rntester.yml b/.github/workflows/visual-approve-rntester.yml new file mode 100644 index 000000000000..9c6f05f7e1e6 --- /dev/null +++ b/.github/workflows/visual-approve-rntester.yml @@ -0,0 +1,67 @@ +# Records a maintainer's approval of a pull request's visual differences when +# the visual-change-approved label is added, and updates the approval gate +# when it is added or removed. Runs from the default branch on +# pull_request_target and never checks out pull request code. +name: Visual Approve RNTester + +on: + pull_request_target: + types: [labeled, unlabeled] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false + +env: + OUT_DIR: /tmp/visual-approval + VISUAL_BUCKET: ${{ vars.VISUAL_BUCKET }} + VISUAL_PUBLIC_BASE_URL: ${{ vars.VISUAL_PUBLIC_BASE_URL }} + +jobs: + approve: + if: github.event.label.name == 'visual-change-approved' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + id-token: write + issues: write + pull-requests: write + steps: + - name: Checkout default branch + uses: actions/checkout@v6 + with: + # Never the pull request: this workflow has write permissions. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Decide + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/approve-run.js prepare + - name: Check for an approval to record + id: staged + run: | + if [[ -d "$OUT_DIR/upload" ]]; then + echo "upload=true" >> "$GITHUB_OUTPUT" + fi + - name: Authenticate to Google Cloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Record approval + if: steps.staged.outputs.upload == 'true' + run: gcloud storage cp --recursive --no-clobber "$OUT_DIR/upload/*" "gs://$VISUAL_BUCKET/" + - name: Update gate + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/approve-run.js report diff --git a/.github/workflows/visual-capture-rntester.yml b/.github/workflows/visual-capture-rntester.yml new file mode 100644 index 000000000000..700ba05ff45c --- /dev/null +++ b/.github/workflows/visual-capture-rntester.yml @@ -0,0 +1,201 @@ +# Untrusted half of RNTester visual regression testing. Builds RNTester from +# the pull request, captures the visual flows on one canonical device per +# platform, and uploads the PNGs plus a manifest as artifacts. It has no +# secrets and no storage access: comparison and publication happen in +# visual-compare-rntester.yml, which runs from the default branch. +name: Visual Capture RNTester + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + MAESTRO_VERSION: 2.6.1 + VISUAL_FLOWS: packages/rn-tester/visual-tests/flows + VISUAL_OUTPUT_DIR: /tmp/visual-capture + VISUAL_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + +jobs: + build_android: + runs-on: ubuntu-latest + container: + image: reactnativecommunity/react-native-android:latest + env: + LC_ALL: C.UTF8 + TERM: 'dumb' + GRADLE_OPTS: '-Dorg.gradle.daemon=false' + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Setup git safe folders + run: git config --global --add safe.directory '*' + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Install node dependencies + uses: ./.github/actions/yarn-install + - name: Setup gradle + uses: ./.github/actions/setup-gradle + - name: Build RNTester release APK + run: | + ./gradlew \ + --init-script "$GITHUB_WORKSPACE/.github/workflow-scripts/legacy-jni-packaging.gradle" \ + -PreactNativeArchitectures=arm64-v8a \ + -Preact.internal.useHermesNightly=true \ + :packages:rn-tester:android:app:assembleRelease + - name: Upload RNTester APK + uses: ./.github/actions/upload-artifact + with: + name: visual-rntester-android + path: packages/rn-tester/android/app/build/outputs/apk/release/ + compression-level: 0 + retention-days: 7 + + capture_android: + needs: build_android + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # The manifest records HEAD^1, the commit this capture is based on. + fetch-depth: 2 + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'zulu' + - name: Install Maestro + run: curl -Ls "https://get.maestro.mobile.dev" | bash + - name: Enable KVM group perms + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Download RNTester APK + uses: actions/download-artifact@v7 + with: + name: visual-rntester-android + path: /tmp/rntester-android + - name: Capture + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 35 + target: google_apis + arch: x86_64 + ram-size: '8192M' + heap-size: '4096M' + disk-size: '10G' + cores: '4' + disable-animations: true + avd-name: visual_emulator + script: | + adb shell settings put system font_scale 1.0 + mkdir -p "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/capture.js android /tmp/rntester-android/app-arm64-v8a-release.apk com.facebook.react.uiapp "$VISUAL_FLOWS" "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/manifest.js android "$VISUAL_OUTPUT_DIR" "$VISUAL_OUTPUT_DIR/manifest.json" + - name: Upload captures + uses: ./.github/actions/upload-artifact + with: + name: visual-capture-android + path: ${{ env.VISUAL_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 14 + - name: Upload Maestro logs + if: failure() + uses: ./.github/actions/upload-artifact + with: + name: visual-maestro-logs-android + path: | + ~/.maestro/tests + visual-capture-report-*.xml + if-no-files-found: ignore + + build_ios: + runs-on: macos-15 + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Build RNTester + uses: ./.github/actions/test-ios-rntester + with: + flavor: Release + use-prebuilds: false + + capture_ios: + needs: build_ios + runs-on: macos-26 + env: + DEVICE_MODEL: iPhone-17-Pro + DEVICE_OS: iOS-26-2 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # The manifest records HEAD^1, the commit this capture is based on. + fetch-depth: 2 + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'zulu' + - name: Install Maestro + run: | + curl -Ls "https://get.maestro.mobile.dev" | bash + brew tap facebook/fb + brew install facebook/fb/idb-companion + - name: Download RNTester app + uses: actions/download-artifact@v7 + with: + name: RNTesterApp-NewArch-Release + path: /tmp/RNTesterBuild/RNTester.app + - name: Boot and configure simulator + id: simulator + run: | + UDID=$(xcrun simctl list devices available -j | jq -r --arg runtime "com.apple.CoreSimulator.SimRuntime.$DEVICE_OS" --arg name "${DEVICE_MODEL//-/ }" '.devices[$runtime][] | select(.name == $name) | .udid' | head -1) + if [[ -z "$UDID" ]]; then + echo "::error::No $DEVICE_MODEL simulator on $DEVICE_OS" + xcrun simctl list devices available + exit 1 + fi + xcrun simctl boot "$UDID" + xcrun simctl bootstatus "$UDID" + xcrun simctl ui "$UDID" appearance light + xcrun simctl ui "$UDID" content_size large + xcrun simctl spawn "$UDID" defaults write -g AppleLocale en_US + xcrun simctl spawn "$UDID" defaults write -g AppleLanguages -array en + echo "udid=$UDID" >> "$GITHUB_OUTPUT" + - name: Capture + run: | + mkdir -p "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/capture.js ios /tmp/RNTesterBuild/RNTester.app com.meta.RNTester.localDevelopment "$VISUAL_FLOWS" "$VISUAL_OUTPUT_DIR" "${{ steps.simulator.outputs.udid }}" + node .github/workflow-scripts/visual/manifest.js ios "$VISUAL_OUTPUT_DIR" "$VISUAL_OUTPUT_DIR/manifest.json" "${{ steps.simulator.outputs.udid }}" + - name: Upload captures + uses: ./.github/actions/upload-artifact + with: + name: visual-capture-ios + path: ${{ env.VISUAL_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 14 + - name: Upload Maestro logs + if: failure() + uses: ./.github/actions/upload-artifact + with: + name: visual-maestro-logs-ios + path: | + ~/.maestro/tests + visual-capture-report-*.xml + if-no-files-found: ignore diff --git a/.github/workflows/visual-compare-rntester.yml b/.github/workflows/visual-compare-rntester.yml new file mode 100644 index 000000000000..1bf4fd120639 --- /dev/null +++ b/.github/workflows/visual-compare-rntester.yml @@ -0,0 +1,99 @@ +# Trusted half of RNTester visual regression testing. Runs from the default +# branch after Visual Capture RNTester completes, including for pull requests +# from forks. The capture artifacts are treated as inert data: they are +# validated before any image is decoded, and nothing in them is executed. +# +# Pull requests: compare with the baseline for the base commit (or its +# nearest ancestor with one), upload a report, and set check runs. +# Pushes to the default branch: as above, then publish the captures as the +# immutable baseline for that commit. +name: Visual Compare RNTester + +on: + workflow_run: + workflows: [Visual Capture RNTester] + types: [completed] + +permissions: + contents: read + +concurrency: + # Serializes baseline publication on the default branch. + group: ${{ github.workflow }}-${{ github.event.workflow_run.event == 'push' && 'main' || github.event.workflow_run.id }} + cancel-in-progress: false + +env: + OUT_DIR: /tmp/visual-out + VISUAL_ARTIFACTS_DIR: /tmp/visual-artifacts + VISUAL_BUCKET: ${{ vars.VISUAL_BUCKET }} + VISUAL_PUBLIC_BASE_URL: ${{ vars.VISUAL_PUBLIC_BASE_URL }} + VISUAL_CONFIG: packages/rn-tester/visual-tests/config.json + +jobs: + compare: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + permissions: + actions: read + checks: write + contents: read + id-token: write + issues: write + pull-requests: write + steps: + - name: Checkout default branch + uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Install node dependencies + uses: ./.github/actions/yarn-install + - name: Download captures + uses: actions/download-artifact@v7 + with: + pattern: visual-capture-* + path: ${{ env.VISUAL_ARTIFACTS_DIR }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Compare + env: + GITHUB_TOKEN: ${{ github.token }} + CAPTURE_EVENT: ${{ github.event.workflow_run.event }} + CAPTURE_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + CAPTURE_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + CAPTURE_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} + CAPTURE_RUN_ID: ${{ github.event.workflow_run.id }} + CAPTURE_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node .github/workflow-scripts/visual/compare-run.js prepare + - name: Check for files to upload + id: staged + run: | + if [[ -d "$OUT_DIR/upload" ]]; then + echo "upload=true" >> "$GITHUB_OUTPUT" + fi + - name: Authenticate to Google Cloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Upload report and baseline + if: steps.staged.outputs.upload == 'true' + run: | + # Objects are write-once; --no-clobber makes a rerun skip what an + # earlier attempt already uploaded. + gcloud storage cp --recursive --no-clobber "$OUT_DIR/upload/*" "gs://$VISUAL_BUCKET/" + if [[ -d "$OUT_DIR/baseline-images" ]]; then + gcloud storage cp --recursive --no-clobber "$OUT_DIR/baseline-images/*" "gs://$VISUAL_BUCKET/" + # A baseline exists once its manifest does, so the manifest goes last. + gcloud storage cp --recursive --no-clobber "$OUT_DIR/baseline-manifests/*" "gs://$VISUAL_BUCKET/" + fi + - name: Report + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/compare-run.js report diff --git a/package.json b/package.json index 1fa53cb0e539..a1743c2892cd 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,8 @@ "micromatch": "^4.0.4", "node-fetch": "^2.2.0", "nullthrows": "^1.1.1", + "pixelmatch": "5.3.0", + "pngjs": "7.0.0", "prettier": "3.9.4", "react": "19.3.0", "react-test-renderer": "19.3.0", diff --git a/packages/rn-tester/js/examples/Text/TextSharedExamples.js b/packages/rn-tester/js/examples/Text/TextSharedExamples.js index 00ed6c2e27bb..f10cacc936fd 100644 --- a/packages/rn-tester/js/examples/Text/TextSharedExamples.js +++ b/packages/rn-tester/js/examples/Text/TextSharedExamples.js @@ -249,7 +249,7 @@ function LastLineClippingExample(): React.Node { // clipped by one physical pixel after layout rounding (issue #53450). The // border makes any clipping of the last line visible. return ( - + {[11, 12, 13, 14, 15, 16, 17].map(fontSize => ( Date: Fri, 16 Oct 2026 07:42:19 +0100 Subject: [PATCH 2/2] Capture visual tests from Test All's RNTester builds Visual capture now runs as a reusable workflow called from Test All, after build_android and test_ios_rntester, and downloads their RNTester Release artifacts instead of building RNTester again. Visual Compare RNTester runs when Test All completes, including when unrelated jobs fail, and only if the run uploaded a capture. The Android emulator uses the pixel_7 device profile, so the screen size and density no longer depend on the emulator action's defaults. The emulator version is pinned too, and is part of the environment key, since emulator updates can change rendering. The comparison downloads each platform's capture by name, since download-artifact extracts a single pattern match without its own directory, which left a one-platform run with nothing compared. A platform with no capture is now reported as such rather than omitted. The capture also dismisses a Pixel Launcher ANR dialog raised during boot, which hid the app in about a third of runs. Adds packages/rn-tester/visual-tests/README.md on adding tests, local runs, approvals and tolerances. Changelog: [Internal] --- .../workflow-scripts/__tests__/visual-test.js | 8 +- .../workflow-scripts/visual/compare-run.js | 32 ++++++- .github/workflow-scripts/visual/manifest.js | 15 ++++ .github/workflows/test-all.yml | 14 +++ .github/workflows/visual-capture-rntester.yml | 89 ++++++------------- .github/workflows/visual-compare-rntester.yml | 52 +++++++++-- packages/rn-tester/README.md | 4 + packages/rn-tester/visual-tests/README.md | 72 +++++++++++++++ 8 files changed, 210 insertions(+), 76 deletions(-) create mode 100644 packages/rn-tester/visual-tests/README.md diff --git a/.github/workflow-scripts/__tests__/visual-test.js b/.github/workflow-scripts/__tests__/visual-test.js index bcecae2a80dc..98e8970e1d1c 100644 --- a/.github/workflow-scripts/__tests__/visual-test.js +++ b/.github/workflow-scripts/__tests__/visual-test.js @@ -51,6 +51,7 @@ const ANDROID_ENVIRONMENT = { fontScale: '1.0', locale: 'en-US', animatorDurationScale: '0.0', + emulatorVersion: '37.2.12', }; function makePng(width, height, paint = () => [255, 255, 255, 255]) { @@ -122,9 +123,12 @@ describe('visual regression', () => { ); }); - test('changes when any keyed property drifts', () => { + test.each([ + ['fontScale', '1.15'], + ['emulatorVersion', '37.3.1'], + ])('changes when %s drifts', (property, value) => { expect( - environmentKey('android', {...ANDROID_ENVIRONMENT, fontScale: '1.15'}), + environmentKey('android', {...ANDROID_ENVIRONMENT, [property]: value}), ).not.toBe(environmentKey('android', ANDROID_ENVIRONMENT)); }); diff --git a/.github/workflow-scripts/visual/compare-run.js b/.github/workflow-scripts/visual/compare-run.js index 3a7bb6ca1b7e..5027ff729538 100644 --- a/.github/workflow-scripts/visual/compare-run.js +++ b/.github/workflow-scripts/visual/compare-run.js @@ -45,6 +45,7 @@ const fs = require('fs'); const path = require('path'); const BASELINE_SEARCH_DEPTH = 20; +const PLATFORMS = ['android', 'ios']; // The default Actions token posts as this bot. Only its comments are edited. const COMMENT_AUTHOR = 'github-actions[bot]'; @@ -203,6 +204,20 @@ async function comparePlatform({ }; } +function missingCapture(platform, context) { + return { + platform, + environmentKey: null, + manifest: null, + baseline: null, + results: [], + error: { + title: 'no capture', + summary: `Test All run ${context.runId} has no ${platform} capture, so there is nothing to compare. Its capture job failed or didn't run: check the run for a build or capture failure.`, + }, + }; +} + function stageBaseline({outDir, captureDir, manifest, sha}) { const prefix = baselinePrefix(manifest.environmentKey, sha); for (const image of manifest.images) { @@ -243,11 +258,15 @@ async function prepare() { const prefix = reportPrefix(context); const platforms = []; - for (const entry of fs.readdirSync(artifactsDir, {withFileTypes: true})) { - if (!entry.isDirectory() || !entry.name.startsWith('visual-capture-')) { + for (const expectedPlatform of PLATFORMS) { + const captureDir = path.join( + artifactsDir, + `visual-capture-${expectedPlatform}`, + ); + if (!fs.existsSync(captureDir)) { + platforms.push(missingCapture(expectedPlatform, context)); continue; } - const captureDir = path.join(artifactsDir, entry.name); const result = await comparePlatform({ github, context, @@ -257,6 +276,11 @@ async function prepare() { outDir, prefix, }); + if (result.platform !== expectedPlatform) { + throw new Error( + `visual-capture-${expectedPlatform}: manifest is for ${result.platform}`, + ); + } if (context.kind === 'main') { stageBaseline({ outDir, @@ -282,7 +306,7 @@ async function prepare() { fingerprint, platforms: platforms.map(({manifest, results, ...platform}) => ({ ...platform, - git: manifest.git, + git: manifest?.git ?? null, results: serializableResults(results), })), }; diff --git a/.github/workflow-scripts/visual/manifest.js b/.github/workflow-scripts/visual/manifest.js index 5bbcfbcaf3f8..6b7832ede2c4 100644 --- a/.github/workflow-scripts/visual/manifest.js +++ b/.github/workflow-scripts/visual/manifest.js @@ -34,6 +34,7 @@ const KEYED_PROPERTIES = { 'fontScale', 'locale', 'animatorDurationScale', + 'emulatorVersion', ], ios: ['runtime', 'deviceType', 'appearance', 'contentSizeCategory', 'locale'], }; @@ -145,6 +146,19 @@ function run(command, args) { .trim(); } +/** The emulator package's version, from the Android SDK. */ +function readEmulatorVersion() { + const sdk = process.env.ANDROID_HOME ?? process.env.ANDROID_SDK_ROOT; + if (sdk == null) { + return null; + } + const properties = fs.readFileSync( + path.join(sdk, 'emulator', 'source.properties'), + 'utf8', + ); + return /^Pkg\.Revision=(\S+)$/m.exec(properties)?.[1] ?? null; +} + function readAndroidEnvironment() { const getprop = name => run('adb', ['shell', 'getprop', name]); const setting = (namespace, name) => @@ -164,6 +178,7 @@ function readAndroidEnvironment() { fontScale: setting('system', 'font_scale'), locale: getprop('persist.sys.locale') || getprop('ro.product.locale'), animatorDurationScale: setting('global', 'animator_duration_scale'), + emulatorVersion: readEmulatorVersion(), }; } diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index fe4f2a56633b..3f4d780d2b2e 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -209,6 +209,13 @@ jobs: fail-on-error: true secrets: inherit + test_visual_ios_rntester: + needs: test_ios_rntester + if: ${{ needs.test_ios_rntester.result == 'success' }} + uses: ./.github/workflows/visual-capture-rntester.yml + with: + platform: ios + test_e2e_ios_templateapp: needs: [build_npm_package, prebuild_apple_dependencies, check_code_changes] if: needs.check_code_changes.outputs.should_test_ios == 'true' @@ -406,6 +413,13 @@ jobs: retry-attempt: 2 secrets: inherit + test_visual_android_rntester: + needs: build_android + if: ${{ needs.build_android.result == 'success' }} + uses: ./.github/workflows/visual-capture-rntester.yml + with: + platform: android + build_npm_package: runs-on: 8-core-ubuntu needs: diff --git a/.github/workflows/visual-capture-rntester.yml b/.github/workflows/visual-capture-rntester.yml index 700ba05ff45c..cde456a4b221 100644 --- a/.github/workflows/visual-capture-rntester.yml +++ b/.github/workflows/visual-capture-rntester.yml @@ -1,24 +1,22 @@ -# Untrusted half of RNTester visual regression testing. Builds RNTester from -# the pull request, captures the visual flows on one canonical device per -# platform, and uploads the PNGs plus a manifest as artifacts. It has no -# secrets and no storage access: comparison and publication happen in -# visual-compare-rntester.yml, which runs from the default branch. +# Untrusted half of RNTester visual regression testing. Called from Test All +# once the RNTester Release build for a platform is ready, it captures the +# visual flows on one canonical device and uploads the PNGs plus a manifest as +# an artifact. It has no secrets and no storage access: comparison and +# publication happen in visual-compare-rntester.yml, which runs from the +# default branch when Test All completes. name: Visual Capture RNTester on: - pull_request: - push: - branches: - - main - workflow_dispatch: + workflow_call: + inputs: + platform: + description: 'The platform to capture, either android or ios' + required: true + type: string permissions: contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - env: MAESTRO_VERSION: 2.6.1 VISUAL_FLOWS: packages/rn-tester/visual-tests/flows @@ -26,42 +24,8 @@ env: VISUAL_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} jobs: - build_android: - runs-on: ubuntu-latest - container: - image: reactnativecommunity/react-native-android:latest - env: - LC_ALL: C.UTF8 - TERM: 'dumb' - GRADLE_OPTS: '-Dorg.gradle.daemon=false' - steps: - - name: Checkout - uses: actions/checkout@v6 - - name: Setup git safe folders - run: git config --global --add safe.directory '*' - - name: Setup node.js - uses: ./.github/actions/setup-node - - name: Install node dependencies - uses: ./.github/actions/yarn-install - - name: Setup gradle - uses: ./.github/actions/setup-gradle - - name: Build RNTester release APK - run: | - ./gradlew \ - --init-script "$GITHUB_WORKSPACE/.github/workflow-scripts/legacy-jni-packaging.gradle" \ - -PreactNativeArchitectures=arm64-v8a \ - -Preact.internal.useHermesNightly=true \ - :packages:rn-tester:android:app:assembleRelease - - name: Upload RNTester APK - uses: ./.github/actions/upload-artifact - with: - name: visual-rntester-android - path: packages/rn-tester/android/app/build/outputs/apk/release/ - compression-level: 0 - retention-days: 7 - capture_android: - needs: build_android + if: inputs.platform == 'android' runs-on: ubuntu-latest steps: - name: Checkout @@ -86,7 +50,7 @@ jobs: - name: Download RNTester APK uses: actions/download-artifact@v7 with: - name: visual-rntester-android + name: rntester-release path: /tmp/rntester-android - name: Capture uses: reactivecircus/android-emulator-runner@v2 @@ -94,6 +58,12 @@ jobs: api-level: 35 target: google_apis arch: x86_64 + # Pins screen size and density, which the action otherwise leaves + # to the AVD defaults. + profile: pixel_7 + # Emulator 37.2.12. Its version can change rendering, so it's + # pinned and is part of the environment key. + emulator-build: '16428233' ram-size: '8192M' heap-size: '4096M' disk-size: '10G' @@ -102,6 +72,12 @@ jobs: avd-name: visual_emulator script: | adb shell settings put system font_scale 1.0 + # Pixel Launcher often stops responding during boot, and its dialog + # covers the app and anything captured beneath it. The setting + # suppresses new dialogs, and stopping the launcher dismisses one + # that's already showing. + adb shell settings put global hide_error_dialogs 1 + adb shell am force-stop "$(adb shell cmd package resolve-activity --brief -a android.intent.action.MAIN -c android.intent.category.HOME | tail -1 | cut -d/ -f1 | tr -d '\r')" mkdir -p "$VISUAL_OUTPUT_DIR" node .github/workflow-scripts/visual/capture.js android /tmp/rntester-android/app-arm64-v8a-release.apk com.facebook.react.uiapp "$VISUAL_FLOWS" "$VISUAL_OUTPUT_DIR" node .github/workflow-scripts/visual/manifest.js android "$VISUAL_OUTPUT_DIR" "$VISUAL_OUTPUT_DIR/manifest.json" @@ -122,19 +98,8 @@ jobs: visual-capture-report-*.xml if-no-files-found: ignore - build_ios: - runs-on: macos-15 - steps: - - name: Checkout - uses: actions/checkout@v6 - - name: Build RNTester - uses: ./.github/actions/test-ios-rntester - with: - flavor: Release - use-prebuilds: false - capture_ios: - needs: build_ios + if: inputs.platform == 'ios' runs-on: macos-26 env: DEVICE_MODEL: iPhone-17-Pro diff --git a/.github/workflows/visual-compare-rntester.yml b/.github/workflows/visual-compare-rntester.yml index 1bf4fd120639..f17981de34f5 100644 --- a/.github/workflows/visual-compare-rntester.yml +++ b/.github/workflows/visual-compare-rntester.yml @@ -1,7 +1,8 @@ # Trusted half of RNTester visual regression testing. Runs from the default -# branch after Visual Capture RNTester completes, including for pull requests -# from forks. The capture artifacts are treated as inert data: they are -# validated before any image is decoded, and nothing in them is executed. +# branch after Test All completes, including for pull requests from forks, and +# compares whatever visual-capture-rntester.yml captured in that run. The +# capture artifacts are treated as inert data: they are validated before any +# image is decoded, and nothing in them is executed. # # Pull requests: compare with the baseline for the base commit (or its # nearest ancestor with one), upload a report, and set check runs. @@ -11,7 +12,7 @@ name: Visual Compare RNTester on: workflow_run: - workflows: [Visual Capture RNTester] + workflows: [Test All] types: [completed] permissions: @@ -30,8 +31,32 @@ env: VISUAL_CONFIG: packages/rn-tester/visual-tests/config.json jobs: + find_captures: + # Test All can fail for reasons unrelated to rendering, so a failed run is + # still compared. A platform with no capture is reported as such. + if: contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-latest + permissions: + actions: read + outputs: + found: ${{ steps.find.outputs.found }} + captures: ${{ steps.find.outputs.captures }} + steps: + - name: Find captures + id: find + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + CAPTURES=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" \ + --jq '.artifacts[] | select(.name | startswith("visual-capture-")) | .name' | tr '\n' ' ') + echo "Captures: ${CAPTURES:-none}" + echo "found=$([[ -n "$CAPTURES" ]] && echo true || echo false)" >> "$GITHUB_OUTPUT" + echo "captures=$CAPTURES" >> "$GITHUB_OUTPUT" + compare: - if: github.event.workflow_run.conclusion == 'success' + needs: find_captures + if: needs.find_captures.outputs.found == 'true' runs-on: ubuntu-latest permissions: actions: read @@ -49,11 +74,22 @@ jobs: uses: ./.github/actions/setup-node - name: Install node dependencies uses: ./.github/actions/yarn-install - - name: Download captures + # Downloaded by name: a pattern that matches one artifact extracts it + # without the per-artifact directory that compare-run.js reads. + - name: Download Android capture + if: contains(needs.find_captures.outputs.captures, 'visual-capture-android') + uses: actions/download-artifact@v7 + with: + name: visual-capture-android + path: ${{ env.VISUAL_ARTIFACTS_DIR }}/visual-capture-android + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Download iOS capture + if: contains(needs.find_captures.outputs.captures, 'visual-capture-ios') uses: actions/download-artifact@v7 with: - pattern: visual-capture-* - path: ${{ env.VISUAL_ARTIFACTS_DIR }} + name: visual-capture-ios + path: ${{ env.VISUAL_ARTIFACTS_DIR }}/visual-capture-ios run-id: ${{ github.event.workflow_run.id }} github-token: ${{ github.token }} - name: Compare diff --git a/packages/rn-tester/README.md b/packages/rn-tester/README.md index b0b3cdb367e7..47c6cc4508d1 100644 --- a/packages/rn-tester/README.md +++ b/packages/rn-tester/README.md @@ -67,3 +67,7 @@ This is different from apps created using `react-native init` which have a depen ## Running tests (iOS) RNTester provides both integration and unit tests. You can run them all by hitting `Cmd+U` in Xcode. Alternatively, it is possible to run just unit tests or integration tests using the keybinding after selecting the appropriate scheme (either `RNTesterUnitTests` or `RNTesterIntegrationTests`). + +## Visual tests + +CI captures screenshots of selected RNTester examples on Android and iOS and compares them with `main`. See [visual-tests/README.md](visual-tests/README.md) to add a test or review a failure. diff --git a/packages/rn-tester/visual-tests/README.md b/packages/rn-tester/visual-tests/README.md new file mode 100644 index 000000000000..bc068b744e52 --- /dev/null +++ b/packages/rn-tester/visual-tests/README.md @@ -0,0 +1,72 @@ +# RNTester visual tests + +Visual tests capture screenshots of RNTester examples in CI and compare them with the screenshots of the same tests on `main`. They catch rendering changes, particularly in text layout, that functional E2E tests don't. + +Each test is a [Maestro](https://maestro.dev) flow in `flows/`. Maestro only opens the example and takes the screenshot - comparison happens afterwards in CI, so a flow never decides whether it passed. + +## How it runs in CI + +1. Test All builds RNTester Release for Android and iOS. Once a build is ready, the `test_visual_android_rntester` and `test_visual_ios_rntester` jobs (`.github/workflows/visual-capture-rntester.yml`) run every flow in this directory on one canonical device per platform: an API 35 Pixel 7 emulator and an iPhone 17 Pro simulator on iOS 26.2. They upload the screenshots and a manifest of the device environment as `visual-capture-android` and `visual-capture-ios` artifacts. +2. When Test All completes, Visual Compare RNTester (`.github/workflows/visual-compare-rntester.yml`) compares the screenshots with the baseline and reports a `Visual regression (android)` and `Visual regression (ios)` check and a `Visual regression approval` gate, plus a single PR comment with the expected, captured and diff images for anything that changed. +3. On pushes to `main`, the captures become the baseline for that commit. + +The baseline for a PR is the one for the commit it's based on, or the nearest earlier commit on `main` that has one. Baselines are keyed by device environment too (OS image, screen size, density, font scale, locale and so on), so a change to the CI device starts a fresh baseline rather than failing every PR. + +## Adding a test + +1. Pick or add an RNTester example, and give its root view a `testID`. Keep the example's content fixed: no dates, random values, network images or animations that are still running when the screenshot is taken. +2. Add a flow to `flows/`. File names must be unique, and each `takeScreenshot` path becomes the test ID, so make it descriptive: + + ```yaml + appId: ${APP_ID} + --- + - runFlow: + file: ../helpers/open-example.yml + env: + EXAMPLE_URL: rntester://example/TextExample/textWidthMode + - extendedWaitUntil: + visible: + id: 'text-width-mode-example' + timeout: 15000 + - takeScreenshot: + path: ${VISUAL_OUTPUT_DIR}/text/width-mode/default + cropOn: + id: 'text-width-mode-example' + ``` + + `open-example.yml` deep-links to `rntester://example//`. Wait for something that's only visible once the example has rendered, and crop to the example's `testID` so that the status bar and surrounding UI aren't part of the screenshot. + +3. Open a PR. The new test has no baseline yet, so the report lists it as new, and like any other difference it needs a maintainer's approval (see below). Once the PR lands, `main` is captured with it and later PRs are compared against that. + +To try a flow locally, build and install RNTester Release on a simulator or emulator and run it with Maestro: + +```sh +maestro test \ + -e APP_ID=com.meta.RNTester.localDevelopment \ + -e VISUAL_OUTPUT_DIR=/tmp/visual \ + packages/rn-tester/visual-tests/flows/text-width-mode.yml +``` + +Use `com.facebook.react.uiapp` as the `APP_ID` on Android. Local screenshots are only useful for checking the flow and the crop - they won't match CI's pixel for pixel unless you use the same device and OS image. + +## When a check fails + +Open the PR comment or the check run and compare the expected, captured and diff images. + +- If the change is unintended, fix it and push. +- If it's intended, a maintainer applies the `visual-change-approved` label. The approval covers the exact set of differences in the report: if a later push changes them, the label is removed and the PR needs a new review. The baseline itself only changes once the PR lands and `main` is captured again. + +## Tolerances + +Comparison is strict by default: any changed pixel is a difference. If a test is genuinely nondeterministic, loosen it for that test in `config.json`, keyed by test ID, and say why in the PR: + +```json +{ + "defaults": {"threshold": 0, "includeAA": true, "maxDiffPixels": 0}, + "tests": { + "text/width-mode/default": {"maxDiffPixels": 20} + } +} +``` + +`threshold` is pixelmatch's per-pixel color distance from 0 to 1, `includeAA` counts anti-aliased pixels, and `maxDiffPixels` and `maxDiffRatio` set how many changed pixels (or what share of the image) can pass. Prefer making the example deterministic over adding a tolerance.