perf: enforce journey budgets and reference traces
This commit is contained in:
209
tests/performance/drive-trace.spec.ts
Normal file
209
tests/performance/drive-trace.spec.ts
Normal file
@@ -0,0 +1,209 @@
|
||||
import { performance as nodePerformance } from 'node:perf_hooks';
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import type { CDPSession, Page } from '@playwright/test';
|
||||
import { expect, test } from '../helpers/serve-dist';
|
||||
import { authorizedPerformanceUrl } from '../helpers/test-build-auth';
|
||||
import {
|
||||
calculateCls,
|
||||
calculateLabTbt,
|
||||
calculateLostPresentationShare,
|
||||
evaluateEventToNextPaint,
|
||||
evaluateSceneContinuity,
|
||||
latestLcpBeforeCutoff,
|
||||
nearestRankP95,
|
||||
postRunningLongTasks,
|
||||
unexpectedLateShifts,
|
||||
type ScriptedActionSample,
|
||||
} from '../../src/testing/performance-instrumentation';
|
||||
|
||||
type NamedProfile = 'desktop-high' | 'mobile-low';
|
||||
|
||||
const profileForProject = (name: string): {
|
||||
name: NamedProfile;
|
||||
tier: 'high' | 'low';
|
||||
viewport: { width: number; height: number };
|
||||
latency: number;
|
||||
download: number;
|
||||
upload: number;
|
||||
sceneCpuP95: number;
|
||||
lostShare: number;
|
||||
} => name === 'performance-mobile'
|
||||
? { name: 'mobile-low', tier: 'low', viewport: { width: 390, height: 844 }, latency: 150, download: 200_000, upload: 93_750, sceneCpuP95: 24, lostShare: 0.1 }
|
||||
: { name: 'desktop-high', tier: 'high', viewport: { width: 1_440, height: 900 }, latency: 40, download: 1_250_000, upload: 1_250_000, sceneCpuP95: 12, lostShare: 0.05 };
|
||||
|
||||
const sleep = (duration: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, duration));
|
||||
|
||||
const stableTargetCenter = async (page: Page, selector: string) => {
|
||||
const locator = page.locator(selector).filter({ visible: true });
|
||||
await expect(locator).toHaveCount(1);
|
||||
await expect(locator).toBeEnabled();
|
||||
const box = await locator.boundingBox();
|
||||
if (box === null) throw new Error(`Trace target is not visible: ${selector}`);
|
||||
return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
||||
};
|
||||
|
||||
const dispatchPointerAction = async (
|
||||
page: Page,
|
||||
cdp: CDPSession,
|
||||
at: (offset: number) => Promise<void>,
|
||||
offset: number,
|
||||
id: string,
|
||||
selector: string,
|
||||
) => {
|
||||
await at(offset - 50);
|
||||
const center = await stableTargetCenter(page, selector);
|
||||
const markerSelector = `[data-drive-target="${id}"]`;
|
||||
await page.locator(selector).filter({ visible: true }).evaluate((element, actionId) => {
|
||||
element.setAttribute('data-drive-target', actionId);
|
||||
}, id);
|
||||
await at(offset - 20);
|
||||
await cdp.send('Input.dispatchMouseEvent', { type: 'mouseMoved', ...center });
|
||||
await at(offset);
|
||||
await page.evaluate(({ actionId, target }) => window.__PORTFOLIO_PERFORMANCE__?.beginAction(actionId, target), { actionId: id, target: markerSelector });
|
||||
await cdp.send('Input.dispatchMouseEvent', { type: 'mousePressed', button: 'left', clickCount: 1, ...center });
|
||||
await at(offset + 40);
|
||||
await cdp.send('Input.dispatchMouseEvent', { type: 'mouseReleased', button: 'left', clickCount: 1, ...center });
|
||||
};
|
||||
|
||||
const dispatchWheelBurst = async (page: Page, cdp: CDPSession, at: (offset: number) => Promise<void>, offset: number, deltaY: number) => {
|
||||
const center = await page.evaluate(() => {
|
||||
const viewport = window.visualViewport;
|
||||
return {
|
||||
x: (viewport?.offsetLeft ?? 0) + (viewport?.width ?? innerWidth) / 2,
|
||||
y: (viewport?.offsetTop ?? 0) + (viewport?.height ?? innerHeight) / 2,
|
||||
};
|
||||
});
|
||||
for (const deltaOffset of [0, 40, 80, 120, 160, 200]) {
|
||||
await at(offset + deltaOffset);
|
||||
await cdp.send('Input.dispatchMouseEvent', { type: 'mouseWheel', deltaX: 0, deltaY, ...center });
|
||||
}
|
||||
};
|
||||
|
||||
test('drive-trace.v1 runs on current Chromium as explicitly non-reference evidence', async ({ page, distUrl }, testInfo) => {
|
||||
test.setTimeout(75_000);
|
||||
const profile = profileForProject(testInfo.project.name);
|
||||
await page.setViewportSize(profile.viewport);
|
||||
const cdp = await page.context().newCDPSession(page);
|
||||
await cdp.send('Network.enable');
|
||||
await cdp.send('Network.clearBrowserCache');
|
||||
await cdp.send('Network.emulateNetworkConditions', {
|
||||
offline: false,
|
||||
latency: profile.latency,
|
||||
downloadThroughput: profile.download,
|
||||
uploadThroughput: profile.upload,
|
||||
});
|
||||
await page.emulateMedia({ reducedMotion: 'no-preference' });
|
||||
await page.goto('about:blank');
|
||||
const loadStarted = nodePerformance.now();
|
||||
await page.goto(authorizedPerformanceUrl(distUrl, profile.tier), { waitUntil: 'load', timeout: 10_000 });
|
||||
const loadDuration = nodePerformance.now() - loadStarted;
|
||||
expect(loadDuration).toBeLessThanOrEqual(10_000);
|
||||
await expect.poll(() => page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.snapshot().lcp.length ?? 0), { timeout: 10_000 }).toBeGreaterThan(0);
|
||||
await expect(page.locator('.journey-world')).toHaveAttribute('data-scene-state', 'running', { timeout: 5_000 });
|
||||
await expect(page.locator('.journey-hud__quality')).toHaveText(profile.tier === 'high' ? 'High' : 'Low');
|
||||
|
||||
const warmupRafs = await page.evaluate(() => new Promise<number[]>((resolve) => {
|
||||
const samples: number[] = [];
|
||||
const frame = (now: number): void => {
|
||||
samples.push(now);
|
||||
if (samples.length === 121) resolve(samples);
|
||||
else requestAnimationFrame(frame);
|
||||
};
|
||||
requestAnimationFrame(frame);
|
||||
}));
|
||||
const warmupIntervals = warmupRafs.slice(1).map((stamp, index) => stamp - warmupRafs[index]!);
|
||||
const warmupMedian = [...warmupIntervals].sort((a, b) => a - b)[Math.floor(warmupIntervals.length / 2)]!;
|
||||
const validReferenceCadence = Math.abs(warmupMedian - 16.667) <= 1;
|
||||
if (process.env.PORTFOLIO_REFERENCE_LAB === '1') expect(validReferenceCadence).toBe(true);
|
||||
|
||||
await page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.startTrace());
|
||||
const nodeTraceStart = nodePerformance.now();
|
||||
const at = async (offset: number): Promise<void> => {
|
||||
const remaining = nodeTraceStart + offset - nodePerformance.now();
|
||||
if (remaining > 0) await sleep(remaining);
|
||||
};
|
||||
|
||||
await dispatchPointerAction(page, cdp, at, 1_000, 'start', 'button:has-text("START THE DRIVE")');
|
||||
await dispatchWheelBurst(page, cdp, at, 1_500, 100);
|
||||
await at(2_350);
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Tab', code: 'Tab', windowsVirtualKeyCode: 9 });
|
||||
await at(2_370);
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Tab', code: 'Tab', windowsVirtualKeyCode: 9 });
|
||||
await expect(page.getByRole('slider', { name: 'Sound volume' })).toBeFocused();
|
||||
await at(2_400);
|
||||
await page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.beginAction('volume', 'input[aria-label="Sound volume"]'));
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'ArrowLeft', code: 'ArrowLeft', windowsVirtualKeyCode: 37 });
|
||||
await at(2_440);
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'ArrowLeft', code: 'ArrowLeft', windowsVirtualKeyCode: 37 });
|
||||
await expect(page.getByRole('slider', { name: 'Sound volume' })).toHaveValue('0.35');
|
||||
await dispatchWheelBurst(page, cdp, at, 2_800, 100);
|
||||
await dispatchPointerAction(page, cdp, at, 3_700, 'mute', 'button:has-text("Mute sound")');
|
||||
await expect(page.getByRole('button', { name: 'Unmute sound' })).toBeVisible();
|
||||
await dispatchWheelBurst(page, cdp, at, 4_100, 100);
|
||||
await dispatchPointerAction(page, cdp, at, 5_000, 'unmute', 'button:has-text("Unmute sound")');
|
||||
await dispatchWheelBurst(page, cdp, at, 5_400, 100);
|
||||
await dispatchPointerAction(page, cdp, at, 6_300, 'pause', 'button:has-text("Pause motion")');
|
||||
await dispatchWheelBurst(page, cdp, at, 6_700, 100);
|
||||
await dispatchPointerAction(page, cdp, at, 7_600, 'resume', 'button:has-text("Resume motion")');
|
||||
await dispatchWheelBurst(page, cdp, at, 8_000, 100);
|
||||
await at(8_800);
|
||||
await page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.armNavigationSettlementBarrier());
|
||||
await dispatchPointerAction(page, cdp, at, 8_900, 'toolkit', 'nav a[href="#toolkit"]');
|
||||
await dispatchWheelBurst(page, cdp, at, 9_300, -100);
|
||||
await dispatchWheelBurst(page, cdp, at, 10_600, -100);
|
||||
await dispatchWheelBurst(page, cdp, at, 11_900, 100);
|
||||
await dispatchWheelBurst(page, cdp, at, 13_200, 100);
|
||||
await dispatchPointerAction(page, cdp, at, 14_500, 'contact', 'nav a[href="#contact"]');
|
||||
await at(20_000);
|
||||
await page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.endTrace());
|
||||
|
||||
await expect(page).toHaveURL(/#contact$/u);
|
||||
await expect(page.locator('#contact > h2')).toBeFocused();
|
||||
await expect(page.locator('.journey-hud__quality')).toHaveText(profile.tier === 'high' ? 'High' : 'Low');
|
||||
const snapshot = await page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.snapshot());
|
||||
if (snapshot === undefined || snapshot.traceStart === null || snapshot.traceEnd === null || snapshot.firstSceneFrame === null) {
|
||||
throw new Error('The authenticated performance snapshot is incomplete.');
|
||||
}
|
||||
const expectedPeriod = warmupMedian;
|
||||
const fcp = await page.evaluate(() => performance.getEntriesByName('first-contentful-paint')[0]?.startTime ?? null);
|
||||
const lcp = latestLcpBeforeCutoff(snapshot.lcp, snapshot.cutoff);
|
||||
const cls = calculateCls(snapshot.layoutShifts);
|
||||
const tbt = calculateLabTbt(snapshot.longTasks, fcp, snapshot.traceStart);
|
||||
const postRunning = postRunningLongTasks(snapshot.longTasks, snapshot.traceStart, snapshot.traceEnd);
|
||||
const eventToPaint = evaluateEventToNextPaint(snapshot.actions as ScriptedActionSample[]);
|
||||
const continuity = evaluateSceneContinuity(snapshot.sceneIntervals, snapshot.sceneFrames, expectedPeriod);
|
||||
const lost = calculateLostPresentationShare(snapshot.pageRafs, expectedPeriod, Math.floor(20_000 / expectedPeriod) - 3);
|
||||
const lateShifts = unexpectedLateShifts(snapshot.layoutShifts, snapshot.firstSceneFrame, snapshot.traceEnd);
|
||||
const sceneCpuP95 = nearestRankP95(snapshot.sceneFrames.map((frame) => frame.end - frame.start));
|
||||
|
||||
const budgetResults = {
|
||||
lcp: lcp <= 3_000,
|
||||
cls: cls <= 0.1,
|
||||
tbt: tbt <= 300,
|
||||
postRunningLongTasks: postRunning.length === 0,
|
||||
eventToNextPaint: eventToPaint.p95 <= 200,
|
||||
sceneCpu: sceneCpuP95 <= profile.sceneCpuP95,
|
||||
continuity: continuity.passes,
|
||||
lostPresentation: lost.hasSufficientCoverage && lost.share <= profile.lostShare,
|
||||
lateShifts: lateShifts.length === 0,
|
||||
};
|
||||
if (process.env.PORTFOLIO_REFERENCE_LAB === '1') {
|
||||
expect(Object.values(budgetResults).every(Boolean)).toBe(true);
|
||||
}
|
||||
expect(snapshot.navigationBarrier.cancelledEpoch).not.toBeNull();
|
||||
|
||||
const report = {
|
||||
schema: 'drive-trace.v1',
|
||||
evidenceKind: 'current-chromium-developer-machine-non-reference',
|
||||
gatingReferenceResult: false,
|
||||
profile: profile.name,
|
||||
tier: profile.tier,
|
||||
browser: testInfo.project.name,
|
||||
metrics: { warmupMedian, lcp, cls, tbt, postRunningLongTaskCount: postRunning.length, eventToNextPaintP95: eventToPaint.p95, sceneCpuP95, continuity, lostPresentationShare: lost.share, lateShiftCount: lateShifts.length },
|
||||
budgetResults,
|
||||
referenceBudgetsPass: validReferenceCadence && Object.values(budgetResults).every(Boolean),
|
||||
validReferenceCadence,
|
||||
pendingPhysicalReferenceRuns: 5,
|
||||
};
|
||||
await writeFile(testInfo.outputPath('drive-trace.v1.json'), `${JSON.stringify(report, null, 2)}\n`);
|
||||
});
|
||||
Reference in New Issue
Block a user