xeno/Portfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: bind performance evidence to production resources

Browse Source
This commit is contained in:
LucaP
2026-09-02 08:53:00 +03:00
parent 5b7e47c2a2
commit e3ac3a4db4
23 changed files with 642 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
tests/performance/aggregation.test.ts
View File

@@ -1,36 +1,67 @@
import { generateKeyPairSync } from 'node:crypto';
import { describe, expect, it } from 'vitest';
import { aggregateFiveDriveTraces } from '../../scripts/lib/performance-aggregation.mjs';
import { signReferenceCandidate } from '../../scripts/lib/reference-evidence.mjs';
const run = (lcp: number, tbt: number, runIndex: number, gatingReferenceResult = true) => ({
const trusted = generateKeyPairSync('ed25519');
const attacker = generateKeyPairSync('ed25519');
const policy = {
schema: 'portfolio-reference-trust-policy/v1',
keyId: 'lab-release-2026',
publicKeyPem: trusted.publicKey.export({ type: 'spki', format: 'pem' }).toString(),
};
const candidate = (lcp: number, tbt: number, runIndex: number) => ({
schema: 'drive-trace.v1',
evidenceKind: 'validated-reference-candidate-unsigned',
gatingReferenceResult: false,
referenceCandidateValidated: true,
profile: 'desktop-high',
tier: 'high',
referenceRun: { runId: `run-${runIndex}`, runIndex, attestationDigest: `${runIndex}`.repeat(64) },
gatingReferenceResult,
validReferenceCadence: true,
referenceBudgetsPass: true,
metrics: { lcp, tbt },
budgetResults: { cls: true, scene: true },
budgetResults: { cls: true, scene: true, runtimeResources: true, requestConcurrency: true },
});
const signed = (lcp: number, tbt: number, runIndex: number) => signReferenceCandidate(
candidate(lcp, tbt, runIndex),
{ keyId: policy.keyId, privateKey: trusted.privateKey },
);
describe('five-run drive trace aggregation', () => {
it('uses worst and median gates across exactly five reference runs', () => {
it('uses worst and median gates across exactly five trusted signed runs', () => {
expect(aggregateFiveDriveTraces([
run(2_000, 100, 1), run(2_100, 110, 2), run(2_500, 200, 3), run(2_900, 250, 4), run(3_000, 300, 5),
])).toMatchObject({
signed(2_000, 100, 1), signed(2_100, 110, 2), signed(2_500, 200, 3), signed(2_900, 250, 4), signed(3_000, 300, 5),
], policy)).toMatchObject({
runCount: 5,
passes: true,
summary: { lcpWorst: 3_000, lcpMedian: 2_500, tbtWorst: 300, tbtMedian: 200 },
});
});
it('rejects non-reference developer evidence and wrong run counts', () => {
expect(aggregateFiveDriveTraces([
run(2_000, 100, 1, false), run(2_000, 100, 2), run(2_000, 100, 3), run(2_000, 100, 4), run(2_000, 100, 5),
]).passes).toBe(false);
expect(() => aggregateFiveDriveTraces([run(2_000, 100, 1)])).toThrow(/five/u);
it('rejects unsigned developer reports and wrong run counts', () => {
expect(() => aggregateFiveDriveTraces([
candidate(2_000, 100, 1), signed(2_000, 100, 2), signed(2_000, 100, 3), signed(2_000, 100, 4), signed(2_000, 100, 5),
], policy)).toThrow(/signature/u);
expect(() => aggregateFiveDriveTraces([signed(2_000, 100, 1)], policy)).toThrow(/five/u);
});
it('rejects duplicate or incomplete authenticated reference identities', () => {
const duplicate = [1, 2, 3, 4, 4].map((index) => run(2_000, 100, index));
expect(() => aggregateFiveDriveTraces(duplicate)).toThrow(/run indexes 1 through 5/u);
it('rejects self-selected keys, tampered measurements, and duplicate replay', () => {
const selfSigned = signReferenceCandidate(candidate(2_000, 100, 1), { keyId: policy.keyId, privateKey: attacker.privateKey });
expect(() => aggregateFiveDriveTraces([
selfSigned, signed(2_000, 100, 2), signed(2_000, 100, 3), signed(2_000, 100, 4), signed(2_000, 100, 5),
], policy)).toThrow(/signature/u);
const tampered = { ...signed(2_000, 100, 1), metrics: { lcp: 1, tbt: 1 } };
expect(() => aggregateFiveDriveTraces([
tampered, signed(2_000, 100, 2), signed(2_000, 100, 3), signed(2_000, 100, 4), signed(2_000, 100, 5),
], policy)).toThrow(/digest|signature/u);
const replay = signed(2_000, 100, 4);
expect(() => aggregateFiveDriveTraces([
signed(2_000, 100, 1), signed(2_000, 100, 2), signed(2_000, 100, 3), replay, replay,
], policy)).toThrow(/run indexes 1 through 5|duplicate/u);
});
});

82
tests/performance/budgets.test.ts
View File

@@ -4,6 +4,9 @@ import {
evaluateBudgetTotals,
evaluateMediaAssetBudget,
evaluateRuntimeBudgetEvidence,
evaluateRuntimeTraceBudgetEvidence,
validateRuntimeBudgetTraceReport,
calculatePeakRequestConcurrency,
PERFORMANCE_BUDGETS,
} from '../../scripts/lib/bundle-budgets.mjs';
@@ -121,4 +124,83 @@ describe('release budgets', () => {
tiers: [{ tier: 'low', textures: [], geometryBytes: 0, renderTargets: [] }],
})).toThrow(/exactly one Low and High/u);
});
it('counts peak concurrency from actual production resource timing intervals', () => {
const peak = calculatePeakRequestConcurrency([
{ name: '/assets/a.webp', initiatorType: 'img', startTime: 0, responseEnd: 50 },
{ name: '/assets/b.webp', initiatorType: 'img', startTime: 10, responseEnd: 60 },
{ name: '/assets/environment-a.js', initiatorType: 'script', startTime: 20, responseEnd: 70 },
{ name: '/assets/a.webm', initiatorType: 'video', startTime: 0, responseEnd: 50 },
{ name: '/assets/b.mp4', initiatorType: 'video', startTime: 10, responseEnd: 60 },
]);
expect(peak).toEqual({ imageScene: 3, video: 2 });
expect(evaluateRuntimeTraceBudgetEvidence({
schema: 'portfolio-runtime-trace-evidence/v1',
peakRequests: peak,
allocation: { tier: 'low', textures: [], geometryBytes: 0, renderTargets: [] },
}).violations.map((item) => item.budget)).toEqual(expect.arrayContaining(['imageSceneConcurrency', 'videoConcurrency']));
});
it('recomputes actual trace allocations and ignores a forged low total', () => {
const result = evaluateRuntimeTraceBudgetEvidence({
schema: 'portfolio-runtime-trace-evidence/v1',
peakRequests: { imageScene: 0, video: 0 },
allocation: {
tier: 'low',
textures: [{ id: 'actual-production-texture', width: 4_096, height: 4_096, bytesPerPixel: 4, layers: 1, mipmapped: true }],
geometryBytes: 0,
renderTargets: [{ id: 'default-framebuffer', width: 390, height: 844, bytesPerPixel: 4, attachments: 2, samples: 1 }],
totalBytes: 1,
},
});
expect(result.tier.totalBytes).toBeGreaterThan(PERFORMANCE_BUDGETS.lowTierGpuEstimate!);
expect(result.violations.map((item) => item.budget)).toEqual(expect.arrayContaining(['mobileTextureDimension', 'lowTierGpuEstimate']));
});
it('accepts only a production-path High capture whose framebuffer matches the observed DPR cap', () => {
const report = {
schema: 'runtime-budget-trace.v1',
evidenceKind: 'authenticated-test-build-production-path',
gatingReferenceResult: false,
profile: 'desktop-high',
tier: 'high',
observed: {
viewport: { width: 1_440, height: 900 },
devicePixelRatio: 2,
rendererPixelRatio: 1.5,
},
runtimeBudgetEvidence: {
schema: 'portfolio-runtime-trace-evidence/v1',
peakRequests: { imageScene: 1, video: 0 },
allocation: {
tier: 'high',
textures: [],
geometryBytes: 0,
renderTargets: [
{ id: 'default-framebuffer', width: 2_160, height: 1_350, bytesPerPixel: 4, attachments: 2, samples: 4 },
],
},
},
};
expect(validateRuntimeBudgetTraceReport(report).violations).toEqual([]);
expect(() => validateRuntimeBudgetTraceReport({
...report,
runtimeBudgetEvidence: {
...report.runtimeBudgetEvidence,
allocation: {
...report.runtimeBudgetEvidence.allocation,
renderTargets: [
{ id: 'default-framebuffer', width: 1_440, height: 900, bytesPerPixel: 4, attachments: 2, samples: 4 },
],
},
},
})).toThrow(/framebuffer dimensions/u);
expect(() => validateRuntimeBudgetTraceReport({
schema: 'portfolio-runtime-budget-evidence/v1',
peakRequests: { imageScene: 0, video: 0 },
tiers: [],
})).toThrow(/authenticated production-path/u);
});
});

21
tests/performance/drive-trace.spec.ts
View File

@@ -4,6 +4,7 @@ import type { CDPSession, Page } from '@playwright/test';
import { expect, test } from '../helpers/serve-dist';
import { authorizedPerformanceUrl } from '../helpers/test-build-auth';
import { loadAuthenticatedReferenceRun, validateObservedReferenceEnvironment } from './reference-run';
import { calculatePeakRequestConcurrency, evaluateRuntimeTraceBudgetEvidence } from '../../scripts/lib/bundle-budgets.mjs';
import {
calculateCls,
calculateLabTbt,
@@ -111,6 +112,7 @@ test('drive-trace.v1 runs on current Chromium with fail-closed reference authent
reducedMotion: matchMedia('(prefers-reduced-motion: reduce)').matches ? 'reduce' as const : 'no-preference' as const,
saveData: Boolean((navigator as Navigator & { connection?: { saveData?: boolean } }).connection?.saveData),
pageZoom: visualViewport?.scale ?? 1,
devicePixelRatio,
}));
validateObservedReferenceEnvironment(referenceRun, {
...observed,
@@ -182,6 +184,7 @@ test('drive-trace.v1 runs on current Chromium with fail-closed reference authent
if (snapshot === undefined || snapshot.traceStart === null || snapshot.traceEnd === null || snapshot.firstSceneFrame === null) {
throw new Error('The authenticated performance snapshot is incomplete.');
}
if (snapshot.runtimeAllocation === null) throw new Error('The production scene did not emit an actual allocation ledger.');
const expectedPeriod = warmupMedian;
const fcp = await page.evaluate(() => performance.getEntriesByName('first-contentful-paint')[0]?.startTime ?? null);
const lcp = latestLcpBeforeCutoff(snapshot.lcp, snapshot.cutoff);
@@ -193,6 +196,15 @@ test('drive-trace.v1 runs on current Chromium with fail-closed reference authent
const lost = calculateLostPresentationShare(snapshot.pageRafs, expectedPeriod, Math.floor(20_000 / expectedPeriod) - 3);
const lateShifts = unexpectedLateShifts(snapshot.layoutShifts, snapshot.firstSceneFrame, snapshot.traceEnd);
const sceneCpuP95 = nearestRankP95(snapshot.sceneFrames.map((frame) => frame.end - frame.start));
const resourceTimings = await page.evaluate(() => performance.getEntriesByType('resource').map((entry) => {
const resource = entry as PerformanceResourceTiming;
return { name: resource.name, initiatorType: resource.initiatorType, startTime: resource.startTime, responseEnd: resource.responseEnd };
}));
const peakRequests = calculatePeakRequestConcurrency(resourceTimings);
const runtimeBudgetEvidence = { schema: 'portfolio-runtime-trace-evidence/v1', peakRequests, allocation: snapshot.runtimeAllocation };
const runtimeBudget = evaluateRuntimeTraceBudgetEvidence(runtimeBudgetEvidence);
const runtimeResourcePass = !runtimeBudget.violations.some((item) => ['mobileTextureDimension', 'lowTierGpuEstimate', 'highTierGpuEstimate'].includes(item.budget));
const requestConcurrencyPass = !runtimeBudget.violations.some((item) => ['imageSceneConcurrency', 'videoConcurrency'].includes(item.budget));
const budgetResults = {
lcp: lcp <= 3_000,
@@ -204,6 +216,8 @@ test('drive-trace.v1 runs on current Chromium with fail-closed reference authent
continuity: continuity.passes,
lostPresentation: lost.hasSufficientCoverage && lost.share <= profile.lostShare,
lateShifts: lateShifts.length === 0,
runtimeResources: runtimeResourcePass,
requestConcurrency: requestConcurrencyPass,
};
if (process.env.PORTFOLIO_REFERENCE_LAB === '1') {
expect(Object.values(budgetResults).every(Boolean)).toBe(true);
@@ -212,13 +226,16 @@ test('drive-trace.v1 runs on current Chromium with fail-closed reference authent
const report = {
schema: 'drive-trace.v1',
evidenceKind: referenceRun === null ? 'current-chromium-developer-machine-non-reference' : 'authenticated-physical-reference',
gatingReferenceResult: referenceRun !== null,
evidenceKind: referenceRun === null ? 'current-chromium-developer-machine-non-reference' : 'validated-reference-candidate-unsigned',
gatingReferenceResult: false,
referenceCandidateValidated: referenceRun !== null,
...(referenceRun === null ? {} : { referenceRun: { runId: referenceRun.runId, runIndex: referenceRun.runIndex, attestationDigest: referenceRun.attestationDigest } }),
profile: profile.name,
tier: profile.tier,
browser: testInfo.project.name,
metrics: { warmupMedian, lcp, cls, tbt, postRunningLongTaskCount: postRunning.length, eventToNextPaintP95: eventToPaint.p95, sceneCpuP95, continuity, lostPresentationShare: lost.share, lateShiftCount: lateShifts.length },
runtimeBudgetEvidence,
runtimeBudgetSummary: runtimeBudget.tier,
budgetResults,
referenceBudgetsPass: validReferenceCadence && Object.values(budgetResults).every(Boolean),
validReferenceCadence,

13
tests/performance/reference-run.test.ts
View File

@@ -75,6 +75,7 @@ describe('authenticated reference runs', () => {
reducedMotion: 'no-preference',
saveData: false,
pageZoom: 1,
devicePixelRatio: 2,
browserName: 'Chromium',
browserVersion: '140.0.0',
startedFromAboutBlank: true,
@@ -85,9 +86,21 @@ describe('authenticated reference runs', () => {
reducedMotion: 'no-preference',
saveData: false,
pageZoom: 1,
devicePixelRatio: 2,
browserName: 'Chromium',
browserVersion: '140.0.0',
startedFromAboutBlank: true,
})).toThrow(/viewport/u);
expect(() => validateObservedReferenceEnvironment(authenticated, {
viewport: { width: 1_440, height: 900 },
visible: true,
reducedMotion: 'no-preference',
saveData: false,
pageZoom: 1,
devicePixelRatio: 1,
browserName: 'Chromium',
browserVersion: '140.0.0',
startedFromAboutBlank: true,
})).toThrow(/pixel ratio/u);
});
});

3
tests/performance/reference-run.ts
View File

@@ -25,6 +25,7 @@ interface ObservedEnvironment {
reducedMotion: 'no-preference' | 'reduce';
saveData: boolean;
pageZoom: number;
devicePixelRatio: number;
browserName: string;
browserVersion: string;
startedFromAboutBlank: boolean;
@@ -98,5 +99,7 @@ export const loadAuthenticatedReferenceRun = async (profile: ReferenceProfile):
export const validateObservedReferenceEnvironment = (reference: AuthenticatedReferenceRun, observed: ObservedEnvironment): void => {
if (observed.viewport.width !== reference.viewport.width || observed.viewport.height !== reference.viewport.height) throw new Error('Observed viewport does not match the attestation.');
if (!observed.visible || observed.reducedMotion !== 'no-preference' || observed.saveData || observed.pageZoom !== 1 || !observed.startedFromAboutBlank) throw new Error('Observed reference environment assertions failed.');
const minimumDpr = reference.profile === 'desktop-high' ? 1.5 : 1.25;
if (!Number.isFinite(observed.devicePixelRatio) || observed.devicePixelRatio < minimumDpr) throw new Error('Observed device pixel ratio cannot exercise the maximum tier cap.');
if (observed.browserName !== 'Chromium' || observed.browserVersion !== reference.browserVersion) throw new Error('Observed browser does not match the attestation.');
};

56
tests/performance/runtime-budget-trace.spec.ts Normal file
View File

@@ -0,0 +1,56 @@
import { writeFile } from 'node:fs/promises';
import { expect, test } from '../helpers/serve-dist';
import { authorizedPerformanceUrl } from '../helpers/test-build-auth';
import {
calculatePeakRequestConcurrency,
validateRuntimeBudgetTraceReport,
} from '../../scripts/lib/bundle-budgets.mjs';
test('runtime-budget-trace.v1 captures the actual High production allocation at its DPR cap', async ({ page, distUrl }, testInfo) => {
test.setTimeout(30_000);
const viewport = { width: 1_440, height: 900 };
await page.setViewportSize(viewport);
await page.emulateMedia({ reducedMotion: 'no-preference' });
await page.goto(authorizedPerformanceUrl(distUrl, 'high'), { waitUntil: 'load', timeout: 10_000 });
await expect(page.locator('.journey-world')).toHaveAttribute('data-scene-state', 'running', { timeout: 5_000 });
await expect.poll(() => page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.snapshot().runtimeAllocation ?? null))
.not.toBeNull();
const snapshot = await page.evaluate(() => window.__PORTFOLIO_PERFORMANCE__?.snapshot());
if (snapshot?.runtimeAllocation === null || snapshot?.runtimeAllocation === undefined) {
throw new Error('The authenticated production scene did not emit an allocation ledger.');
}
const resourceTimings = await page.evaluate(() => performance.getEntriesByType('resource').map((entry) => {
const resource = entry as PerformanceResourceTiming;
return {
name: resource.name,
initiatorType: resource.initiatorType,
startTime: resource.startTime,
responseEnd: resource.responseEnd,
};
}));
const report = {
schema: 'runtime-budget-trace.v1',
evidenceKind: 'authenticated-test-build-production-path',
gatingReferenceResult: false,
profile: 'desktop-high',
tier: 'high',
observed: {
viewport,
devicePixelRatio: await page.evaluate(() => devicePixelRatio),
rendererPixelRatio: 1.5,
},
runtimeBudgetEvidence: {
schema: 'portfolio-runtime-trace-evidence/v1',
peakRequests: calculatePeakRequestConcurrency(resourceTimings),
allocation: snapshot.runtimeAllocation,
},
};
const evaluated = validateRuntimeBudgetTraceReport(report);
expect(evaluated.violations).toEqual([]);
await writeFile(testInfo.outputPath('runtime-budget-trace.v1.json'), `${JSON.stringify({
...report,
runtimeBudgetSummary: evaluated.tier,
violations: evaluated.violations,
}, null, 2)}\n`);
});