fix: close performance evidence and budget gates

This commit is contained in:
2026-09-02 08:27:21 +03:00
parent 1734003172
commit 5525557c3a
13 changed files with 541 additions and 39 deletions

View File

@@ -1,9 +1,10 @@
import { describe, expect, it } from 'vitest';
import { aggregateFiveDriveTraces } from '../../scripts/lib/performance-aggregation.mjs';
const run = (lcp: number, tbt: number, gatingReferenceResult = true) => ({
const run = (lcp: number, tbt: number, runIndex: number, gatingReferenceResult = true) => ({
schema: 'drive-trace.v1',
profile: 'desktop-high',
referenceRun: { runId: `run-${runIndex}`, runIndex, attestationDigest: `${runIndex}`.repeat(64) },
gatingReferenceResult,
validReferenceCadence: true,
metrics: { lcp, tbt },
@@ -13,7 +14,7 @@ const run = (lcp: number, tbt: number, gatingReferenceResult = true) => ({
describe('five-run drive trace aggregation', () => {
it('uses worst and median gates across exactly five reference runs', () => {
expect(aggregateFiveDriveTraces([
run(2_000, 100), run(2_100, 110), run(2_500, 200), run(2_900, 250), run(3_000, 300),
run(2_000, 100, 1), run(2_100, 110, 2), run(2_500, 200, 3), run(2_900, 250, 4), run(3_000, 300, 5),
])).toMatchObject({
runCount: 5,
passes: true,
@@ -23,8 +24,13 @@ describe('five-run drive trace aggregation', () => {
it('rejects non-reference developer evidence and wrong run counts', () => {
expect(aggregateFiveDriveTraces([
run(2_000, 100, false), run(2_000, 100), run(2_000, 100), run(2_000, 100), run(2_000, 100),
run(2_000, 100, 1, false), run(2_000, 100, 2), run(2_000, 100, 3), run(2_000, 100, 4), run(2_000, 100, 5),
]).passes).toBe(false);
expect(() => aggregateFiveDriveTraces([run(2_000, 100)])).toThrow(/five/u);
expect(() => aggregateFiveDriveTraces([run(2_000, 100, 1)])).toThrow(/five/u);
});
it('rejects duplicate or incomplete authenticated reference identities', () => {
const duplicate = [1, 2, 3, 4, 4].map((index) => run(2_000, 100, index));
expect(() => aggregateFiveDriveTraces(duplicate)).toThrow(/run indexes 1 through 5/u);
});
});

View File

@@ -1,5 +1,11 @@
import { describe, expect, it } from 'vitest';
import { classifyArtifact, evaluateBudgetTotals, PERFORMANCE_BUDGETS } from '../../scripts/lib/bundle-budgets.mjs';
import {
classifyArtifact,
evaluateBudgetTotals,
evaluateMediaAssetBudget,
evaluateRuntimeBudgetEvidence,
PERFORMANCE_BUDGETS,
} from '../../scripts/lib/bundle-budgets.mjs';
describe('release budgets', () => {
it('pins every exact section 14 threshold', () => {
@@ -31,6 +37,7 @@ describe('release budgets', () => {
expect(classifyArtifact('assets/index-a.css', '')).toBe('critical-css');
expect(classifyArtifact('assets/index-a.js', 'import("./environment-b.js")')).toBe('initial-application-js');
expect(classifyArtifact('assets/environment-b.js', 'three scene')).toBe('first-live-scene');
expect(classifyArtifact('assets/high-tier-b.js', 'optional bloom')).toBe('high-tier-extras');
expect(() => classifyArtifact('assets/mystery.bin', '')).toThrow(/unclassified/u);
});
@@ -38,4 +45,80 @@ describe('release budgets', () => {
expect(evaluateBudgetTotals({ initialHtmlCriticalCss: 100 * 1024 }).violations).toEqual([]);
expect(evaluateBudgetTotals({ initialHtmlCriticalCss: 100 * 1024 + 1 }).violations).toContainEqual(expect.objectContaining({ budget: 'initialHtmlCriticalCss' }));
});
it('rejects an undersized transfer when its video delivery rendition is 20s at 1920x1080', () => {
const result = evaluateMediaAssetBudget({
id: 'project-demo',
mediaType: 'video/webm',
width: 1_920,
height: 1_080,
durationMs: 20_000,
}, {
mediaType: 'video/webm',
width: 1_920,
height: 1_080,
durationMs: 20_000,
}, 1_024);
expect(result.violations.map((violation) => violation.budget)).toEqual(expect.arrayContaining([
'projectVideoDurationSeconds',
'projectVideoWidth',
'projectVideoHeight',
]));
});
it('fails closed when non-empty media dimensions cannot be deterministically probed', () => {
expect(() => evaluateMediaAssetBudget({
id: 'project-still',
mediaType: 'image/webp',
width: 1_280,
height: 720,
}, null, 1_024)).toThrow(/probe evidence/u);
});
it('rejects excessive high-tier extras and request concurrency', () => {
expect(evaluateBudgetTotals({ highTierExtras: PERFORMANCE_BUDGETS.highTierExtras! + 1 }).violations)
.toContainEqual(expect.objectContaining({ budget: 'highTierExtras' }));
expect(evaluateRuntimeBudgetEvidence({
schema: 'portfolio-runtime-budget-evidence/v1',
peakRequests: { imageScene: 3, video: 2 },
tiers: [
{ tier: 'low', textures: [], geometryBytes: 0, renderTargets: [] },
{ tier: 'high', textures: [], geometryBytes: 0, renderTargets: [] },
],
}).violations.map((violation) => violation.budget)).toEqual(expect.arrayContaining([
'imageSceneConcurrency',
'videoConcurrency',
]));
});
it('derives decoded texture and GPU bytes from allocation formulas', () => {
const result = evaluateRuntimeBudgetEvidence({
schema: 'portfolio-runtime-budget-evidence/v1',
peakRequests: { imageScene: 0, video: 0 },
tiers: [
{
tier: 'low',
textures: [{ id: 'oversized', width: 4_096, height: 4_096, bytesPerPixel: 4, layers: 1, mipmapped: true }],
geometryBytes: 40 * 1024 * 1024,
renderTargets: [{ id: 'framebuffer', width: 2_048, height: 2_048, bytesPerPixel: 8, attachments: 2, samples: 1 }],
},
{ tier: 'high', textures: [], geometryBytes: 0, renderTargets: [] },
],
});
expect(result.tiers[0]?.textureBytes).toBe(Math.ceil(4_096 * 4_096 * 4 * 4 / 3));
expect(result.violations.map((violation) => violation.budget)).toEqual(expect.arrayContaining([
'mobileTextureDimension',
'lowTierGpuEstimate',
]));
});
it('fails closed when either named tier allocation ledger is absent', () => {
expect(() => evaluateRuntimeBudgetEvidence({
schema: 'portfolio-runtime-budget-evidence/v1',
peakRequests: { imageScene: 0, video: 0 },
tiers: [{ tier: 'low', textures: [], geometryBytes: 0, renderTargets: [] }],
})).toThrow(/exactly one Low and High/u);
});
});

View File

@@ -3,6 +3,7 @@ import { writeFile } from 'node:fs/promises';
import type { CDPSession, Page } from '@playwright/test';
import { expect, test } from '../helpers/serve-dist';
import { authorizedPerformanceUrl } from '../helpers/test-build-auth';
import { loadAuthenticatedReferenceRun, validateObservedReferenceEnvironment } from './reference-run';
import {
calculateCls,
calculateLabTbt,
@@ -79,9 +80,10 @@ const dispatchWheelBurst = async (page: Page, cdp: CDPSession, at: (offset: numb
}
};
test('drive-trace.v1 runs on current Chromium as explicitly non-reference evidence', async ({ page, distUrl }, testInfo) => {
test('drive-trace.v1 runs on current Chromium with fail-closed reference authentication', async ({ browser, page, distUrl }, testInfo) => {
test.setTimeout(75_000);
const profile = profileForProject(testInfo.project.name);
const referenceRun = await loadAuthenticatedReferenceRun(profile.name);
await page.setViewportSize(profile.viewport);
const cdp = await page.context().newCDPSession(page);
await cdp.send('Network.enable');
@@ -94,6 +96,7 @@ test('drive-trace.v1 runs on current Chromium as explicitly non-reference eviden
});
await page.emulateMedia({ reducedMotion: 'no-preference' });
await page.goto('about:blank');
const startedFromAboutBlank = page.url() === 'about:blank';
const loadStarted = nodePerformance.now();
await page.goto(authorizedPerformanceUrl(distUrl, profile.tier), { waitUntil: 'load', timeout: 10_000 });
const loadDuration = nodePerformance.now() - loadStarted;
@@ -101,6 +104,21 @@ test('drive-trace.v1 runs on current Chromium as explicitly non-reference eviden
await expect.poll(() => page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.snapshot().lcp.length ?? 0), { timeout: 10_000 }).toBeGreaterThan(0);
await expect(page.locator('.journey-world')).toHaveAttribute('data-scene-state', 'running', { timeout: 5_000 });
await expect(page.locator('.journey-hud__quality')).toHaveText(profile.tier === 'high' ? 'High' : 'Low');
if (referenceRun !== null) {
const observed = await page.evaluate(() => ({
viewport: { width: innerWidth, height: innerHeight },
visible: document.visibilityState === 'visible',
reducedMotion: matchMedia('(prefers-reduced-motion: reduce)').matches ? 'reduce' as const : 'no-preference' as const,
saveData: Boolean((navigator as Navigator & { connection?: { saveData?: boolean } }).connection?.saveData),
pageZoom: visualViewport?.scale ?? 1,
}));
validateObservedReferenceEnvironment(referenceRun, {
...observed,
browserName: 'Chromium',
browserVersion: browser.version(),
startedFromAboutBlank,
});
}
const warmupRafs = await page.evaluate(() => new Promise<number[]>((resolve) => {
const samples: number[] = [];
@@ -194,8 +212,9 @@ test('drive-trace.v1 runs on current Chromium as explicitly non-reference eviden
const report = {
schema: 'drive-trace.v1',
evidenceKind: 'current-chromium-developer-machine-non-reference',
gatingReferenceResult: false,
evidenceKind: referenceRun === null ? 'current-chromium-developer-machine-non-reference' : 'authenticated-physical-reference',
gatingReferenceResult: referenceRun !== null,
...(referenceRun === null ? {} : { referenceRun: { runId: referenceRun.runId, runIndex: referenceRun.runIndex, attestationDigest: referenceRun.attestationDigest } }),
profile: profile.name,
tier: profile.tier,
browser: testInfo.project.name,
@@ -203,7 +222,7 @@ test('drive-trace.v1 runs on current Chromium as explicitly non-reference eviden
budgetResults,
referenceBudgetsPass: validReferenceCadence && Object.values(budgetResults).every(Boolean),
validReferenceCadence,
pendingPhysicalReferenceRuns: 5,
pendingPhysicalReferenceRuns: referenceRun === null ? 5 : 0,
};
await writeFile(testInfo.outputPath('drive-trace.v1.json'), `${JSON.stringify(report, null, 2)}\n`);
});

View File

@@ -0,0 +1,93 @@
import { createHash } from 'node:crypto';
import { describe, expect, it } from 'vitest';
import { authenticateReferenceRun, validateObservedReferenceEnvironment } from './reference-run';
const attestation = (runIndex = 1) => ({
schema: 'portfolio-reference-run-attestation/v1',
profile: 'desktop-high',
runId: `desktop-reference-${runIndex}`,
capturedAt: '2026-09-02T05:00:00.000Z',
runIndex,
device: { model: 'Apple M1 MacBook Air', cpu: 'Apple M1', gpu: 'Apple M1 integrated GPU', ramGb: 8 },
browser: { name: 'Chromium', version: '140.0.0', driverVersion: 'Playwright 1.62.1' },
osVersion: 'macOS 15.6',
display: { refreshHz: 60 },
viewport: { width: 1_440, height: 900 },
network: { downstreamKbps: 10_000, upstreamKbps: 10_000, roundTripLatencyMs: 40 },
assertions: {
batteryPercent: 80,
powerSaving: false,
coolStart: true,
noOtherForegroundApplications: true,
cpuEmulation: false,
pageZoom: 1,
reducedMotion: 'no-preference',
saveData: false,
audioState: 'idle',
freshBrowserProfile: true,
startedFromAboutBlank: true,
storageCleared: { serviceWorkers: true, cacheStorage: true, httpCache: true, cookies: true, localStorage: true, sessionStorage: true, indexedDb: true, preferences: true },
physicalReferenceDevice: true,
},
});
describe('authenticated reference runs', () => {
it('cannot mark an ordinary developer trace as gating evidence', () => {
const bytes = Buffer.from(JSON.stringify(attestation()));
expect(() => authenticateReferenceRun({
referenceMode: false,
expectedProfile: 'desktop-high',
attestationBytes: bytes,
expectedDigest: createHash('sha256').update(bytes).digest('hex'),
})).toThrow(/reference mode/u);
});
it('fails closed on digest, device, network, or assertion mismatch', () => {
const invalid = attestation();
invalid.network.downstreamKbps = 9_999;
const bytes = Buffer.from(JSON.stringify(invalid));
expect(() => authenticateReferenceRun({
referenceMode: true,
expectedProfile: 'desktop-high',
attestationBytes: bytes,
expectedDigest: createHash('sha256').update(bytes).digest('hex'),
})).toThrow(/network/u);
expect(() => authenticateReferenceRun({
referenceMode: true,
expectedProfile: 'desktop-high',
attestationBytes: bytes,
expectedDigest: '0'.repeat(64),
})).toThrow(/digest/u);
});
it('authenticates only exact pinned metadata and validates browser-observed assertions', () => {
const bytes = Buffer.from(JSON.stringify(attestation()));
const authenticated = authenticateReferenceRun({
referenceMode: true,
expectedProfile: 'desktop-high',
attestationBytes: bytes,
expectedDigest: createHash('sha256').update(bytes).digest('hex'),
});
expect(authenticated).toMatchObject({ runId: 'desktop-reference-1', runIndex: 1, profile: 'desktop-high' });
expect(() => validateObservedReferenceEnvironment(authenticated, {
viewport: { width: 1_440, height: 900 },
visible: true,
reducedMotion: 'no-preference',
saveData: false,
pageZoom: 1,
browserName: 'Chromium',
browserVersion: '140.0.0',
startedFromAboutBlank: true,
})).not.toThrow();
expect(() => validateObservedReferenceEnvironment(authenticated, {
viewport: { width: 1_440, height: 899 },
visible: true,
reducedMotion: 'no-preference',
saveData: false,
pageZoom: 1,
browserName: 'Chromium',
browserVersion: '140.0.0',
startedFromAboutBlank: true,
})).toThrow(/viewport/u);
});
});

View File

@@ -0,0 +1,102 @@
import { createHash } from 'node:crypto';
import { readFile } from 'node:fs/promises';
export type ReferenceProfile = 'desktop-high' | 'mobile-low';
export interface AuthenticatedReferenceRun {
profile: ReferenceProfile;
runId: string;
runIndex: number;
attestationDigest: string;
browserVersion: string;
viewport: { width: number; height: number };
}
interface AuthenticationInput {
referenceMode: boolean;
expectedProfile: ReferenceProfile;
attestationBytes: Uint8Array;
expectedDigest: string;
}
interface ObservedEnvironment {
viewport: { width: number; height: number };
visible: boolean;
reducedMotion: 'no-preference' | 'reduce';
saveData: boolean;
pageZoom: number;
browserName: string;
browserVersion: string;
startedFromAboutBlank: boolean;
}
const exactKeys = (value: unknown, keys: readonly string[], label: string): Record<string, unknown> => {
if (value === null || typeof value !== 'object' || Array.isArray(value)) throw new Error(`${label} is invalid.`);
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`${label} fields are invalid.`);
return value as Record<string, unknown>;
};
const pinned = {
'desktop-high': { viewport: { width: 1_440, height: 900 }, cpu: 'Apple M1', gpu: 'Apple M1 integrated GPU', ramGb: 8, downstreamKbps: 10_000, upstreamKbps: 10_000, roundTripLatencyMs: 40 },
'mobile-low': { viewport: { width: 390, height: 844 }, cpu: 'Google Tensor', gpu: 'Mali-G78 MP20', ramGb: 8, downstreamKbps: 1_600, upstreamKbps: 750, roundTripLatencyMs: 150 },
} as const;
export const authenticateReferenceRun = (input: AuthenticationInput): AuthenticatedReferenceRun => {
if (!input.referenceMode) throw new Error('Authenticated reference mode is required.');
if (!/^[a-f0-9]{64}$/u.test(input.expectedDigest)) throw new Error('Reference attestation digest is invalid.');
const actualDigest = createHash('sha256').update(input.attestationBytes).digest('hex');
if (actualDigest !== input.expectedDigest) throw new Error('Reference attestation digest mismatch.');
let decoded: unknown;
try { decoded = JSON.parse(Buffer.from(input.attestationBytes).toString('utf8')); } catch { throw new Error('Reference attestation JSON is invalid.'); }
const root = exactKeys(decoded, ['schema', 'profile', 'runId', 'runIndex', 'capturedAt', 'device', 'browser', 'osVersion', 'display', 'viewport', 'network', 'assertions'], 'Reference attestation');
if (root.schema !== 'portfolio-reference-run-attestation/v1' || root.profile !== input.expectedProfile
|| typeof root.runId !== 'string' || !/^[a-z0-9][a-z0-9-]{2,79}$/u.test(root.runId)
|| !Number.isInteger(root.runIndex) || Number(root.runIndex) < 1 || Number(root.runIndex) > 5
|| typeof root.capturedAt !== 'string' || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(root.capturedAt)
|| !Number.isFinite(Date.parse(root.capturedAt))
|| typeof root.osVersion !== 'string' || root.osVersion.length < 2) throw new Error('Reference run metadata is invalid.');
const profile = pinned[input.expectedProfile];
const device = exactKeys(root.device, ['model', 'cpu', 'gpu', 'ramGb'], 'Reference device');
if (typeof device.model !== 'string' || device.model.length < 2 || device.cpu !== profile.cpu || device.gpu !== profile.gpu || device.ramGb !== profile.ramGb) throw new Error('Reference device does not match the pinned class.');
const browser = exactKeys(root.browser, ['name', 'version', 'driverVersion'], 'Reference browser');
if (browser.name !== 'Chromium' || typeof browser.version !== 'string' || !/^\d+(?:\.\d+){1,3}$/u.test(browser.version)
|| typeof browser.driverVersion !== 'string' || browser.driverVersion.length < 2) throw new Error('Reference browser metadata is invalid.');
const display = exactKeys(root.display, ['refreshHz'], 'Reference display');
if (display.refreshHz !== 60) throw new Error('Reference display must be fixed at 60 Hz.');
const viewport = exactKeys(root.viewport, ['width', 'height'], 'Reference viewport');
if (viewport.width !== profile.viewport.width || viewport.height !== profile.viewport.height) throw new Error('Reference viewport does not match the pinned profile.');
const network = exactKeys(root.network, ['downstreamKbps', 'upstreamKbps', 'roundTripLatencyMs'], 'Reference network');
if (network.downstreamKbps !== profile.downstreamKbps || network.upstreamKbps !== profile.upstreamKbps || network.roundTripLatencyMs !== profile.roundTripLatencyMs) throw new Error('Reference network does not match the pinned profile.');
const assertions = exactKeys(root.assertions, ['batteryPercent', 'powerSaving', 'coolStart', 'noOtherForegroundApplications', 'cpuEmulation', 'pageZoom', 'reducedMotion', 'saveData', 'audioState', 'freshBrowserProfile', 'startedFromAboutBlank', 'storageCleared', 'physicalReferenceDevice'], 'Reference assertions');
const storage = exactKeys(assertions.storageCleared, ['serviceWorkers', 'cacheStorage', 'httpCache', 'cookies', 'localStorage', 'sessionStorage', 'indexedDb', 'preferences'], 'Reference storage assertions');
if (typeof assertions.batteryPercent !== 'number' || assertions.batteryPercent <= 50 || assertions.batteryPercent > 100
|| assertions.powerSaving !== false || assertions.coolStart !== true || assertions.noOtherForegroundApplications !== true
|| assertions.cpuEmulation !== false || assertions.pageZoom !== 1 || assertions.reducedMotion !== 'no-preference'
|| assertions.saveData !== false || assertions.audioState !== 'idle' || assertions.freshBrowserProfile !== true
|| assertions.startedFromAboutBlank !== true || assertions.physicalReferenceDevice !== true
|| Object.values(storage).some((value) => value !== true)) throw new Error('Reference environment assertions are not all satisfied.');
return {
profile: input.expectedProfile,
runId: root.runId,
runIndex: Number(root.runIndex),
attestationDigest: actualDigest,
browserVersion: browser.version,
viewport: profile.viewport,
};
};
export const loadAuthenticatedReferenceRun = async (profile: ReferenceProfile): Promise<AuthenticatedReferenceRun | null> => {
if (process.env.PORTFOLIO_REFERENCE_LAB !== '1') return null;
const path = process.env.PORTFOLIO_REFERENCE_ATTESTATION_PATH;
const digest = process.env.PORTFOLIO_REFERENCE_ATTESTATION_SHA256;
if (path === undefined || digest === undefined) throw new Error('Reference mode requires an attestation path and SHA-256 digest.');
return authenticateReferenceRun({ referenceMode: true, expectedProfile: profile, attestationBytes: await readFile(path), expectedDigest: digest });
};
export const validateObservedReferenceEnvironment = (reference: AuthenticatedReferenceRun, observed: ObservedEnvironment): void => {
if (observed.viewport.width !== reference.viewport.width || observed.viewport.height !== reference.viewport.height) throw new Error('Observed viewport does not match the attestation.');
if (!observed.visible || observed.reducedMotion !== 'no-preference' || observed.saveData || observed.pageZoom !== 1 || !observed.startedFromAboutBlank) throw new Error('Observed reference environment assertions failed.');
if (observed.browserName !== 'Chromium' || observed.browserVersion !== reference.browserVersion) throw new Error('Observed browser does not match the attestation.');
};

View File

@@ -98,7 +98,7 @@ describe('scene render quality', () => {
const medium = renderQualityFor('medium', 3, false);
const low = renderQualityFor('low', 3, false);
expect(high).toEqual({ pixelRatio: 2, terrainSegments: 96, particleCount: 384, postEffects: true });
expect(high).toEqual({ pixelRatio: 1.5, terrainSegments: 96, particleCount: 384, postEffects: true });
expect(medium).toEqual({ pixelRatio: 1.5, terrainSegments: 64, particleCount: 192, postEffects: true });
expect(low).toEqual({ pixelRatio: 1.25, terrainSegments: 32, particleCount: 64, postEffects: false });
});