fix: recover and fence atomic activation

This commit is contained in:
2026-09-02 11:49:49 +03:00
parent 79e5d04395
commit a8fa09e21e
12 changed files with 1733 additions and 543 deletions

View File

@@ -1,141 +1,394 @@
import { lstat, mkdtemp, mkdir, readFile, readlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, test } from 'vitest';
import {
lstat,
mkdtemp,
mkdir,
readFile,
readlink,
rename,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import { buildReleasePackage } from '../../scripts/build-release.mjs';
import { activateRelease, cleanupOrphanStages, prepareRelease, readControlState, rollbackRelease } from '../../scripts/deploy/lib/release-transaction.mjs';
import { createReleaseFixture, initializeReleaseRoot } from './release-test-fixture.ts';
import { canonicalValueBytes } from "../../src/security/canonical-digest.ts";
import {
buildReleasePackage,
buildReleasePackageForTest,
} from "../../scripts/build-release.mjs";
import {
activateRelease,
activateReleaseForTest,
cleanupOrphanStages,
prepareRelease,
prepareReleaseForTest,
readControlState,
rollbackReleaseForTest,
} from "../../scripts/deploy/lib/release-transaction.mjs";
import {
createReleaseFixture,
initializeReleaseRoot,
} from "./release-test-fixture.ts";
type Fixture = Awaited<ReturnType<typeof createReleaseFixture>>;
function buildInput(fixture: Fixture, outputPath: string) {
return {
distDir: fixture.distDir, signedManifestPath: fixture.signedManifestPath, sbomPath: fixture.sbomPath,
verifierModulePath: fixture.verifierModulePath, verifySignedReleaseEnvelope: fixture.verifySignedReleaseEnvelope,
trustPolicyPath: fixture.trustPolicyPath, protectedState: fixture.protectedState,
now: '2026-09-02T01:00:00Z', outputPath,
distDir: fixture.distDir,
signedManifestPath: fixture.signedManifestPath,
sbomPath: fixture.sbomPath,
trustPolicyPath: fixture.trustPolicyPath,
verifySignedReleaseEnvelope: fixture.verifySignedReleaseEnvelope,
protectedState: fixture.protectedState,
now: "2026-09-02T01:00:00Z",
outputPath,
};
}
function transactionInput(root: string, archivePath: string, transactionId: string, fixture: Fixture) {
function transactionInput(
root: string,
archivePath: string,
transactionId: string,
fixture: Fixture,
) {
return {
root, archivePath, transactionId, verifierModulePath: fixture.verifierModulePath,
verifySignedReleaseEnvelope: fixture.verifySignedReleaseEnvelope, trustPolicyPath: fixture.trustPolicyPath,
uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0, now: '2026-09-02T01:00:00Z',
root,
archivePath,
transactionId,
testOnlyVerification: {
trustPolicyPath: fixture.trustPolicyPath,
verifySignedReleaseEnvelope: fixture.verifySignedReleaseEnvelope,
},
testClock: () => "2026-09-02T01:00:00Z",
smokeCheck: async ({ releaseDir }: { releaseDir: string }) => {
await readFile(join(releaseDir, 'index.html'));
return Buffer.from('internal-nginx-smoke-ok\n');
await readFile(join(releaseDir, "index.html"));
return Buffer.from("internal-nginx-smoke-ok\n");
},
};
}
describe('fenced atomic Nginx release transaction', () => {
test('prepares immutably without changing current, then activates with one atomic pointer swap', async () => {
const temp = await mkdtemp(join(tmpdir(), 'portfolio-atomic-'));
const fixture = await createReleaseFixture(join(temp, 'fixture'));
const root = join(temp, 'release-root'); await initializeReleaseRoot(root);
const archive = join(temp, 'release.tar'); await buildReleasePackage(buildInput(fixture, archive));
await prepareRelease(transactionInput(root, archive, 'tx-a', fixture));
await expect(lstat(join(root, 'current'))).rejects.toThrow();
await activateRelease({ ...transactionInput(root, archive, 'tx-a', fixture), expectedCurrentManifestDigest: null, expectedFence: 1, now: '2026-09-02T01:01:00Z' });
const target = await readlink(join(root, 'current'));
async function setup(prefix: string) {
const temp = await mkdtemp(join(tmpdir(), prefix));
const fixture = await createReleaseFixture(join(temp, "fixture"));
const root = join(temp, "release-root");
await initializeReleaseRoot(
root,
fixture.protectedState,
fixture.trustPolicy,
);
const archive = join(temp, "release.tar");
await buildReleasePackageForTest(buildInput(fixture, archive));
return { temp, fixture, root, archive };
}
describe("fenced atomic Nginx release transaction", () => {
test("prepares immutably without changing current, then activates with one atomic pointer swap", async () => {
const { fixture, root, archive } = await setup("portfolio-atomic-");
const input = transactionInput(root, archive, "tx-a", fixture);
await prepareReleaseForTest(input);
await expect(lstat(join(root, "current"))).rejects.toThrow();
await activateReleaseForTest({
...input,
expectedCurrentManifestDigest: null,
expectedFence: 1,
});
const target = await readlink(join(root, "current"));
expect(target).toBe(`releases/release-a-${fixture.manifestDigest}/public`);
expect((await lstat(join(root, target, 'index.html'))).mode & 0o777).toBe(0o444);
expect((await lstat(join(root, target, "index.html"))).mode & 0o777).toBe(
0o444,
);
expect((await lstat(join(root, target))).mode & 0o777).toBe(0o555);
expect((await lstat(join(root, target, "index.html"))).uid).toBe(
(await lstat(root)).uid,
);
});
test('rejects stale predecessor/fence, expiry crossing, and concurrent activation without moving the old pointer', async () => {
const temp = await mkdtemp(join(tmpdir(), 'portfolio-fence-'));
const fixture = await createReleaseFixture(join(temp, 'fixture'));
const root = join(temp, 'release-root'); await initializeReleaseRoot(root);
const archive = join(temp, 'release.tar'); await buildReleasePackage(buildInput(fixture, archive));
const input = transactionInput(root, archive, 'tx-a', fixture);
await prepareRelease(input);
await expect(activateRelease({ ...input, expectedCurrentManifestDigest: null, expectedFence: 2, now: '2026-09-02T01:01:00Z' })).rejects.toThrow('fence');
await expect(lstat(join(root, 'current'))).rejects.toThrow();
await expect(activateRelease({ ...input, expectedCurrentManifestDigest: fixture.manifestDigest, expectedFence: 1, now: '2026-09-02T01:01:00Z' })).rejects.toThrow('predecessor');
await expect(activateRelease({ ...input, expectedCurrentManifestDigest: null, expectedFence: 1, now: fixture.expiresAt })).rejects.toThrow('expired');
test("rejects stale predecessor/fence, expiry crossing, and concurrent activation without moving the old pointer", async () => {
const { fixture, root, archive } = await setup("portfolio-fence-");
const input = transactionInput(root, archive, "tx-a", fixture);
await prepareReleaseForTest(input);
await expect(
activateReleaseForTest({
...input,
expectedCurrentManifestDigest: null,
expectedFence: 2,
}),
).rejects.toThrow("fence");
await expect(
activateReleaseForTest({
...input,
expectedCurrentManifestDigest: fixture.manifestDigest,
expectedFence: 1,
}),
).rejects.toThrow("predecessor");
await expect(
activateReleaseForTest({
...input,
expectedCurrentManifestDigest: null,
expectedFence: 1,
testClock: () => fixture.expiresAt,
}),
).rejects.toThrow("expired");
const attempts = await Promise.allSettled([
activateRelease({ ...input, expectedCurrentManifestDigest: null, expectedFence: 1, now: '2026-09-02T01:01:00Z' }),
activateRelease({ ...input, expectedCurrentManifestDigest: null, expectedFence: 1, now: '2026-09-02T01:01:00Z' }),
activateReleaseForTest({
...input,
expectedCurrentManifestDigest: null,
expectedFence: 1,
}),
activateReleaseForTest({
...input,
expectedCurrentManifestDigest: null,
expectedFence: 1,
}),
]);
expect(attempts.filter((result) => result.status === 'fulfilled')).toHaveLength(1);
expect(attempts.filter((result) => result.status === 'rejected')).toHaveLength(1);
expect(await readlink(join(root, 'current'))).toBe(`releases/release-a-${fixture.manifestDigest}/public`);
expect(
attempts.filter((result) => result.status === "fulfilled"),
).toHaveLength(1);
expect(
attempts.filter((result) => result.status === "rejected"),
).toHaveLength(1);
});
test('rechecks trusted time after smoke so an expiry crossing never becomes ready', async () => {
const temp = await mkdtemp(join(tmpdir(), 'portfolio-expiry-crossing-'));
const fixture = await createReleaseFixture(join(temp, 'fixture'));
const root = join(temp, 'release-root'); await initializeReleaseRoot(root);
const archive = join(temp, 'release.tar'); await buildReleasePackage(buildInput(fixture, archive));
const times = ['2026-09-02T01:00:00Z', fixture.expiresAt];
await expect(prepareRelease({ ...transactionInput(root, archive, 'tx-a', fixture), clock: () => times.shift() ?? fixture.expiresAt })).rejects.toThrow('expired while staging');
test("rechecks trusted time after smoke so an expiry crossing never becomes ready", async () => {
const { fixture, root, archive } = await setup(
"portfolio-expiry-crossing-",
);
const times = ["2026-09-02T01:00:00Z", fixture.expiresAt];
await expect(
prepareReleaseForTest({
...transactionInput(root, archive, "tx-a", fixture),
testClock: () => times.shift() ?? fixture.expiresAt,
}),
).rejects.toThrow("expired while staging");
expect((await readControlState(root)).activeTransaction).toBeNull();
await expect(lstat(join(root, 'current'))).rejects.toThrow();
await expect(lstat(join(root, "current"))).rejects.toThrow();
});
test('a crash before swap leaves the predecessor visible and retry after a post-swap crash converges on the committed release', async () => {
const temp = await mkdtemp(join(tmpdir(), 'portfolio-crash-'));
const fixture = await createReleaseFixture(join(temp, 'fixture'));
const root = join(temp, 'release-root'); await initializeReleaseRoot(root);
const archive = join(temp, 'release.tar'); await buildReleasePackage(buildInput(fixture, archive));
const input = { ...transactionInput(root, archive, 'tx-a', fixture), expectedCurrentManifestDigest: null, expectedFence: 1, now: '2026-09-02T01:01:00Z' };
await prepareRelease(input);
await expect(activateRelease({ ...input, crashPoint: 'before-swap' })).rejects.toThrow('before pointer');
await expect(lstat(join(root, 'current'))).rejects.toThrow();
await expect(activateRelease({ ...input, crashPoint: 'after-swap' })).rejects.toThrow('after pointer');
expect(await readlink(join(root, 'current'))).toBe(`releases/release-a-${fixture.manifestDigest}/public`);
const recovered = await activateRelease(input);
test.each([
"after-intent",
"after-extract",
"after-immutable",
"after-release-rename",
"after-receipt",
] as const)(
"recovers an interrupted preparation at %s and safely retries",
async (testPreparationCrashPoint) => {
const { fixture, root, archive } = await setup(
`portfolio-prepare-crash-${testPreparationCrashPoint}-`,
);
const input = transactionInput(root, archive, "tx-a", fixture);
await expect(
prepareReleaseForTest({ ...input, testPreparationCrashPoint }),
).rejects.toThrow("process death");
expect((await readControlState(root)).activeTransaction?.phase).toBe(
"preparing",
);
await prepareReleaseForTest(input);
expect((await readControlState(root)).activeTransaction?.phase).toBe(
"ready",
);
await expect(lstat(join(root, "current"))).rejects.toThrow();
},
);
test("a crash before swap leaves the predecessor visible and retry after a post-swap crash converges", async () => {
const { fixture, root, archive } = await setup(
"portfolio-activation-crash-",
);
const input = {
...transactionInput(root, archive, "tx-a", fixture),
expectedCurrentManifestDigest: null,
expectedFence: 1,
};
await prepareReleaseForTest(input);
await expect(
activateReleaseForTest({ ...input, crashPoint: "before-swap" }),
).rejects.toThrow("before pointer");
await expect(lstat(join(root, "current"))).rejects.toThrow();
await expect(
activateReleaseForTest({ ...input, crashPoint: "after-swap" }),
).rejects.toThrow("after pointer");
expect(await readlink(join(root, "current"))).toBe(
`releases/release-a-${fixture.manifestDigest}/public`,
);
const recovered = await activateReleaseForTest(input);
expect(recovered.recoveredAfterPointerSwap).toBe(true);
expect((await readControlState(root)).currentManifestDigest).toBe(fixture.manifestDigest);
expect((await readControlState(root)).currentManifestDigest).toBe(
fixture.manifestDigest,
);
});
test('rollback requires a fresh signed manifest and never replays the selected old manifest', async () => {
const temp = await mkdtemp(join(tmpdir(), 'portfolio-rollback-'));
const fixtureA = await createReleaseFixture(join(temp, 'fixture-a'));
const root = join(temp, 'release-root'); await initializeReleaseRoot(root);
const archiveA = join(temp, 'a.tar'); await buildReleasePackage(buildInput(fixtureA, archiveA));
const inputA = transactionInput(root, archiveA, 'tx-a', fixtureA);
await prepareRelease(inputA);
await activateRelease({ ...inputA, expectedCurrentManifestDigest: null, expectedFence: 1, now: '2026-09-02T01:01:00Z' });
const releaseA = (await readlink(join(root, 'current'))).split('/')[1] as string;
test("a protected ledger revocation racing final publication is preserved and blocks activation", async () => {
const { fixture, root, archive } = await setup(
"portfolio-revocation-race-",
);
const input = transactionInput(root, archive, "tx-a", fixture);
await prepareReleaseForTest(input);
await expect(
activateReleaseForTest({
...input,
expectedCurrentManifestDigest: null,
expectedFence: 1,
testBeforePointerSwap: async () => {
const state = await readControlState(root);
await writeFile(
join(root, "control/state.next"),
canonicalValueBytes({
...state,
latestLedgerHeadHash: "a".repeat(64),
latestLedgerRevision: state.latestLedgerRevision + 1,
}),
);
await rename(
join(root, "control/state.next"),
join(root, "control/state.json"),
);
},
}),
).rejects.toThrow("authority changed");
expect((await readControlState(root)).latestLedgerRevision).toBe(
fixture.protectedState.latestLedgerRevision + 1,
);
await expect(lstat(join(root, "current"))).rejects.toThrow();
});
const fixtureB = await createReleaseFixture(join(temp, 'fixture-b'));
fixtureB.protectedState.currentManifestDigest = fixtureA.manifestDigest; fixtureB.protectedState.publisherFence = 1;
await writeFile(join(fixtureB.distDir, 'index.html'), '<!doctype html><title>Portfolio B</title>\n');
const signedB = await fixtureB.signFor({ buildId: 'release-b', distDir: fixtureB.distDir, predecessorManifestDigest: fixtureA.manifestDigest, publisherFence: 1, outputPath: fixtureB.signedManifestPath });
const archiveB = join(temp, 'b.tar'); await buildReleasePackage({ ...buildInput(fixtureB, archiveB), now: '2026-09-02T02:00:00Z' });
const inputB = { ...transactionInput(root, archiveB, 'tx-b', fixtureB), now: '2026-09-02T02:00:00Z' };
await prepareRelease(inputB);
await activateRelease({ ...inputB, expectedCurrentManifestDigest: fixtureA.manifestDigest, expectedFence: 2, now: '2026-09-02T02:01:00Z' });
test("rollback requires a fresh signed manifest that binds the selected old artifact closure", async () => {
const { temp, fixture, root, archive } = await setup("portfolio-rollback-");
const inputA = transactionInput(root, archive, "tx-a", fixture);
await prepareReleaseForTest(inputA);
await activateReleaseForTest({
...inputA,
expectedCurrentManifestDigest: null,
expectedFence: 1,
});
const releaseA = (await readlink(join(root, "current"))).split(
"/",
)[1] as string;
await expect(rollbackRelease({ ...inputA, targetReleaseId: releaseA, transactionId: 'tx-replay', expectedCurrentManifestDigest: signedB.manifestDigest, expectedFence: 3, now: '2026-09-02T03:00:00Z' })).rejects.toThrow();
fixture.protectedState.currentManifestDigest = fixture.manifestDigest;
fixture.protectedState.publisherFence = 1;
await writeFile(
join(fixture.distDir, "index.html"),
"<!doctype html><title>Portfolio B</title>\n",
);
const signedB = await fixture.signFor({
buildId: "release-b",
distDir: fixture.distDir,
predecessorManifestDigest: fixture.manifestDigest,
publisherFence: 1,
outputPath: fixture.signedManifestPath,
});
const archiveB = join(temp, "b.tar");
await buildReleasePackageForTest({
...buildInput(fixture, archiveB),
now: "2026-09-02T02:00:00Z",
});
const inputB = {
...transactionInput(root, archiveB, "tx-b", fixture),
testClock: () => "2026-09-02T02:00:00Z",
};
await prepareReleaseForTest(inputB);
await activateReleaseForTest({
...inputB,
expectedCurrentManifestDigest: fixture.manifestDigest,
expectedFence: 2,
});
fixtureA.protectedState.currentManifestDigest = signedB.manifestDigest; fixtureA.protectedState.publisherFence = 2;
const freshSigned = join(temp, 'rollback-signed.json');
const fresh = await fixtureA.signFor({ buildId: 'rollback-a', distDir: fixtureA.distDir, predecessorManifestDigest: signedB.manifestDigest, publisherFence: 2, outputPath: freshSigned });
fixtureA.signedManifestPath = freshSigned;
const rollbackArchive = join(temp, 'rollback.tar'); await buildReleasePackage({ ...buildInput(fixtureA, rollbackArchive), now: '2026-09-02T03:00:00Z' });
const rollback = await rollbackRelease({ ...transactionInput(root, rollbackArchive, 'tx-rollback', fixtureA), targetReleaseId: releaseA,
expectedCurrentManifestDigest: signedB.manifestDigest, expectedFence: 3, now: '2026-09-02T03:00:00Z' });
await expect(
rollbackReleaseForTest({
...inputA,
targetReleaseId: releaseA,
transactionId: "tx-replay",
expectedCurrentManifestDigest: signedB.manifestDigest,
expectedFence: 3,
}),
).rejects.toThrow();
fixture.protectedState.currentManifestDigest = signedB.manifestDigest;
fixture.protectedState.publisherFence = 2;
await writeFile(
join(fixture.distDir, "index.html"),
"<!doctype html><title>Portfolio</title>\n",
);
const freshSigned = join(temp, "rollback-signed.json");
const fresh = await fixture.signFor({
buildId: "rollback-a",
distDir: fixture.distDir,
predecessorManifestDigest: signedB.manifestDigest,
publisherFence: 2,
outputPath: freshSigned,
});
fixture.signedManifestPath = freshSigned;
const rollbackArchive = join(temp, "rollback.tar");
await buildReleasePackageForTest({
...buildInput(fixture, rollbackArchive),
now: "2026-09-02T03:00:00Z",
});
const rollback = await rollbackReleaseForTest({
...transactionInput(root, rollbackArchive, "tx-rollback", fixture),
targetReleaseId: releaseA,
expectedCurrentManifestDigest: signedB.manifestDigest,
expectedFence: 3,
testClock: () => "2026-09-02T03:00:00Z",
});
expect(rollback.manifestDigest).toBe(fresh.manifestDigest);
expect(await readlink(join(root, 'current'))).toContain('rollback-a-');
expect(await readlink(join(root, "current"))).toContain("rollback-a-");
});
test('orphan cleanup and all mutations remain confined to a validated explicit release root', async () => {
const temp = await mkdtemp(join(tmpdir(), 'portfolio-cleanup-'));
const outside = join(temp, 'outside.txt'); await writeFile(outside, 'keep');
const invalidRoot = join(temp, 'not-a-release-root'); await mkdir(join(invalidRoot, 'staging', 'orphan.partial'), { recursive: true });
await expect(cleanupOrphanStages(invalidRoot)).rejects.toThrow('sentinel');
expect(await readFile(outside, 'utf8')).toBe('keep');
const root = join(temp, 'release-root'); await initializeReleaseRoot(root);
await mkdir(join(root, 'staging', 'orphan.partial'), { recursive: true });
await writeFile(join(root, 'staging', 'orphan.partial', 'marker'), 'orphan');
expect(await cleanupOrphanStages(root)).toEqual(['orphan.partial']);
await expect(lstat(join(root, 'staging', 'orphan.partial'))).rejects.toThrow();
expect(await readFile(outside, 'utf8')).toBe('keep');
await expect(cleanupOrphanStages('/')).rejects.toThrow('Broad');
test("production APIs reject caller authority fields and archives outside the upload-only inbox", async () => {
const { fixture, root, archive } = await setup(
"portfolio-production-authority-",
);
await expect(
prepareRelease({
...transactionInput(root, archive, "tx-a", fixture),
smokeCommand: "/bin/true",
} as any),
).rejects.toThrow("unknown");
await expect(
prepareRelease({
root,
archivePath: archive,
transactionId: "tx-a",
smokeCommand: "/bin/true",
}),
).rejects.toThrow("upload-only inbox");
await expect(
activateRelease({
root,
transactionId: "tx-a",
expectedCurrentManifestDigest: null,
expectedFence: 1,
now: "2026-09-02T01:00:00Z",
} as any),
).rejects.toThrow("unknown");
await expect(
buildReleasePackage(buildInput(fixture, join(root, "bad.tar")) as any),
).rejects.toThrow("rejects caller");
});
test("orphan cleanup and all mutations remain confined to a validated explicit release root", async () => {
const temp = await mkdtemp(join(tmpdir(), "portfolio-cleanup-"));
const outside = join(temp, "outside.txt");
await writeFile(outside, "keep");
const invalidRoot = join(temp, "not-a-release-root");
await mkdir(join(invalidRoot, "staging", "orphan.partial"), {
recursive: true,
});
await expect(cleanupOrphanStages(invalidRoot)).rejects.toThrow("sentinel");
expect(await readFile(outside, "utf8")).toBe("keep");
const root = join(temp, "release-root");
await initializeReleaseRoot(root);
await mkdir(join(root, "staging", "orphan.partial"), { recursive: true });
await writeFile(
join(root, "staging", "orphan.partial", "marker"),
"orphan",
);
expect(await cleanupOrphanStages(root)).toEqual(["orphan.partial"]);
await expect(
lstat(join(root, "staging", "orphan.partial")),
).rejects.toThrow();
expect(await readFile(outside, "utf8")).toBe("keep");
await expect(cleanupOrphanStages("/")).rejects.toThrow("Broad");
});
});