import { spawn } from "node:child_process"; import { chmod, lstat, mkdir, open, readFile, readlink, readdir, rename, rm, symlink, unlink, } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { byteDigest, canonicalJsonDigest, canonicalValueBytes, parseCanonicalJson, } from "../../../src/security/canonical-digest.ts"; import { assertDigest, assertReleaseRoot, assertTimestamp, assertToken, atomicCanonicalWrite, exactObject, makeImmutable, safeRemoveStage, verifySignedRelease, writeArchiveFiles, verifySignedReleaseForTest, } from "./release-common.mjs"; import { verifyStagedRelease, verifyStagedReleaseForTest, } from "../verify-staged-release.mjs"; const stateFields = [ "schema", "currentManifestDigest", "currentReleaseId", "publisherFence", "latestLedgerHeadHash", "latestLedgerRevision", "releaseTrustPolicyDigest", "activeTransaction", ]; const transactionFields = [ "transactionId", "manifestDigest", "releaseId", "expectedFence", "phase", "readinessReceiptDigest", ]; const readinessFields = [ "schema", "transactionId", "manifestDigest", "releaseId", "predecessorManifestDigest", "expectedFence", "latestLedgerHeadHash", "latestLedgerRevision", "expiresAt", "packageDigest", "smokeEvidenceDigest", "createdAt", ]; function nowFrom(input) { const value = typeof input === "function" ? input() : (input ?? new Date().toISOString()); return assertTimestamp(value, "trusted current time"); } function normalizeExpectedDigest(value) { if (value === null || value === "none") return null; return assertDigest(value, "expected current manifest digest"); } function protectedStateFrom(state) { return { latestLedgerHeadHash: state.latestLedgerHeadHash, latestLedgerRevision: state.latestLedgerRevision, currentManifestDigest: state.currentManifestDigest, publisherFence: state.publisherFence, releaseTrustPolicyDigest: state.releaseTrustPolicyDigest, }; } async function syncDirectory(path) { const handle = await open(path, "r"); try { await handle.sync(); } finally { await handle.close(); } } async function durableCanonicalWrite(path, value, mode = 0o600) { await atomicCanonicalWrite(path, value, mode); await syncDirectory(dirname(path)); } async function recoverInterruptedPreparation(root, state) { const tx = state.activeTransaction; if (tx === null || tx.phase !== "preparing") return state; const target = `releases/${tx.releaseId}/public`; if ((await currentTarget(root)) === target) throw new Error( "Preparing transaction unexpectedly controls the public pointer; privileged recovery is required.", ); const partialDir = join(root, "staging", `${tx.transactionId}.partial`); const releaseDir = join(root, "releases", tx.releaseId); for (const path of [partialDir, releaseDir]) { if ( await lstat(path).catch((error) => error?.code === "ENOENT" ? null : Promise.reject(error), ) ) { await makeWritable(path); await rm(path, { recursive: true, force: true }); } } await unlink( join(root, "receipts", "readiness", `${tx.transactionId}.json`), ).catch((error) => { if (error?.code !== "ENOENT") throw error; }); const recovered = { ...state, activeTransaction: null }; await durableCanonicalWrite(join(root, "control/state.json"), recovered); return recovered; } export function validateControlState(value) { const state = exactObject(value, stateFields, "release control state"); if (state.schema !== "portfolio-release-control-state/v1") throw new Error("Release control state schema is invalid."); if (state.currentManifestDigest !== null) assertDigest(state.currentManifestDigest, "current manifest digest"); if (state.currentReleaseId !== null) assertToken(state.currentReleaseId, "current release ID"); if ( !Number.isSafeInteger(state.publisherFence) || state.publisherFence < 0 || !Number.isSafeInteger(state.latestLedgerRevision) || state.latestLedgerRevision < 0 ) throw new Error("Release control counters are invalid."); assertDigest(state.latestLedgerHeadHash, "live ledger head"); assertDigest(state.releaseTrustPolicyDigest, "release trust-policy digest"); if ( (state.currentManifestDigest === null) !== (state.currentReleaseId === null) ) throw new Error("Current release pointer state is incoherent."); if (state.activeTransaction !== null) { const tx = exactObject( state.activeTransaction, transactionFields, "active release transaction", ); assertToken(tx.transactionId, "transaction ID"); assertDigest(tx.manifestDigest, "transaction manifest digest"); assertToken(tx.releaseId, "transaction release ID"); if ( !Number.isSafeInteger(tx.expectedFence) || tx.expectedFence !== state.publisherFence + 1 || !["preparing", "ready", "committing"].includes(tx.phase) ) throw new Error("Active transaction fence or phase is invalid."); if (tx.readinessReceiptDigest !== null) assertDigest(tx.readinessReceiptDigest, "readiness receipt digest"); } return state; } export async function readControlState(root) { return validateControlState( parseCanonicalJson(await readFile(join(root, "control/state.json"))), ); } async function withExclusiveLock(root, callback) { const lockPath = join(root, "control", "transaction.lock"); let handle; for (let attempt = 0; attempt < 200; attempt += 1) { try { handle = await open(lockPath, "wx", 0o600); await handle.writeFile(`${process.pid}\n`); await handle.sync(); break; } catch (error) { if (error?.code !== "EEXIST") throw error; const owner = await readFile(lockPath, "utf8").catch(() => ""); if (!/^\d+\n$/u.test(owner)) throw new Error( "Release transaction lock is malformed; protected recovery is required.", ); try { process.kill(Number(owner.trim()), 0); } catch (probeError) { if (probeError?.code !== "ESRCH") throw probeError; await unlink(lockPath).catch((unlinkError) => { if (unlinkError?.code !== "ENOENT") throw unlinkError; }); continue; } await new Promise((resolvePromise) => setTimeout(resolvePromise, 5)); } } if (handle === undefined) throw new Error("Could not obtain the exclusive release transaction lock."); try { return await callback(); } finally { await handle.close(); await unlink(lockPath).catch(() => undefined); } } async function runSmokeCommand(command, releaseDir, manifestDigest) { if (typeof command !== "string" || !isAbsolute(command)) throw new Error("Smoke command must be an absolute executable path."); const info = await lstat(command); if (!info.isFile() || info.isSymbolicLink() || (info.mode & 0o111) === 0) throw new Error("Smoke command is not an executable regular file."); return await new Promise((resolvePromise, reject) => { const child = spawn( command, ["--release-dir", releaseDir, "--manifest-digest", manifestDigest], { shell: false, stdio: ["ignore", "pipe", "pipe"] }, ); const stdout = []; const stderr = []; const timer = setTimeout(() => { child.kill("SIGKILL"); reject(new Error("Staged smoke check timed out.")); }, 15_000); child.stdout.on("data", (chunk) => stdout.push(chunk)); child.stderr.on("data", (chunk) => stderr.push(chunk)); child.once("error", (error) => { clearTimeout(timer); reject(error); }); child.once("close", (code) => { clearTimeout(timer); if (code !== 0) reject( new Error( `Staged smoke check failed: ${Buffer.concat(stderr).toString("utf8").slice(0, 200)}`, ), ); else resolvePromise(Buffer.concat(stdout)); }); }); } async function prepareUnlocked(input) { const root = await assertReleaseRoot(input.root); const transactionId = assertToken(input.transactionId, "transaction ID"); const statePath = join(root, "control/state.json"); let state = await recoverInterruptedPreparation( root, await readControlState(root), ); if (state.activeTransaction !== null) throw new Error("Another release transaction is already active."); const firstNow = nowFrom(input.testClock); const verificationInput = { archivePath: resolve(input.archivePath), protectedState: protectedStateFrom(state), }; const verified = input.testOnlyVerification ? await verifyStagedReleaseForTest({ ...verificationInput, ...input.testOnlyVerification, now: firstNow, }) : await verifyStagedRelease({ ...verificationInput, authorityRoot: root }); const { index, admission, files } = verified; if (input.rollbackTargetReleaseId !== undefined) { const targetId = assertToken( input.rollbackTargetReleaseId, "rollback target release ID", ); const targetIndex = parseCanonicalJson( await readFile( join(root, "releases", targetId, ".release/package-index.json"), ), ); if (targetIndex.manifestDigest === index.manifestDigest) throw new Error( "Rollback must use a newly signed manifest, not replay an old one.", ); const left = canonicalValueBytes(targetIndex.artifacts); const right = canonicalValueBytes(index.artifacts); if (!Buffer.from(left).equals(Buffer.from(right))) throw new Error( "Rollback package does not bind the selected immutable release artifact closure.", ); } const activeTransaction = { transactionId, manifestDigest: admission.manifestDigest, releaseId: index.releaseId, expectedFence: admission.expectedFence, phase: "preparing", readinessReceiptDigest: null, }; await durableCanonicalWrite(statePath, { ...state, activeTransaction }); if (input.testPreparationCrashPoint === "after-intent") throw Object.assign( new Error("Injected preparation process death after intent."), { simulatedProcessDeath: true }, ); const partialName = `${transactionId}.partial`; const partialDir = join(root, "staging", partialName); const releaseDir = join(root, "releases", index.releaseId); try { await mkdir(join(root, "staging"), { recursive: true, mode: 0o750 }); await mkdir(join(root, "releases"), { recursive: true, mode: 0o750 }); await mkdir(join(root, "receipts", "readiness"), { recursive: true, mode: 0o750, }); await safeRemoveStage(root, partialName); await writeArchiveFiles( files.map((file) => file.path.startsWith(".release/") ? file : { ...file, path: `public/${file.path}` }, ), partialDir, ); if (input.testPreparationCrashPoint === "after-extract") throw Object.assign( new Error("Injected preparation process death after extract."), { simulatedProcessDeath: true }, ); const owner = await lstat(root); await makeImmutable(partialDir, owner.uid, owner.gid); if (input.testPreparationCrashPoint === "after-immutable") throw Object.assign( new Error("Injected preparation process death after immutable."), { simulatedProcessDeath: true }, ); const publicDir = join(partialDir, "public"); const smokeBytes = input.smokeCheck !== undefined ? await input.smokeCheck({ releaseDir: publicDir, manifestDigest: admission.manifestDigest, }) : await runSmokeCommand( input.smokeCommand, publicDir, admission.manifestDigest, ); const secondNow = nowFrom(input.testClock); if (Date.parse(secondNow) >= Date.parse(admission.expiresAt)) throw new Error("Release expired while staging or smoke-checking."); const stateAfterSmoke = await readControlState(root); if ( stateAfterSmoke.latestLedgerHeadHash !== admission.latestLedgerHeadHash || stateAfterSmoke.latestLedgerRevision !== admission.latestLedgerRevision || stateAfterSmoke.currentManifestDigest !== admission.predecessorManifestDigest || stateAfterSmoke.publisherFence + 1 !== admission.expectedFence ) { throw new Error("Protected state changed while staging the release."); } await chmod(partialDir, 0o755); await rename(partialDir, releaseDir).catch((error) => { if (error?.code === "EEXIST") throw new Error("Immutable release ID already exists."); throw error; }); await chmod(releaseDir, 0o555); await syncDirectory(join(root, "releases")); if (input.testPreparationCrashPoint === "after-release-rename") throw Object.assign( new Error("Injected preparation process death after release rename."), { simulatedProcessDeath: true }, ); const receipt = { schema: "portfolio-release-readiness-receipt/v1", transactionId, manifestDigest: admission.manifestDigest, releaseId: index.releaseId, predecessorManifestDigest: admission.predecessorManifestDigest, expectedFence: admission.expectedFence, latestLedgerHeadHash: admission.latestLedgerHeadHash, latestLedgerRevision: admission.latestLedgerRevision, expiresAt: admission.expiresAt, packageDigest: byteDigest(await readFile(input.archivePath)), smokeEvidenceDigest: byteDigest( smokeBytes instanceof Uint8Array ? smokeBytes : Buffer.from(String(smokeBytes)), ), createdAt: secondNow, }; const receiptDigest = canonicalJsonDigest( "portfolio-release-readiness-receipt/v1", receipt, ); const receiptPath = join( root, "receipts", "readiness", `${transactionId}.json`, ); await durableCanonicalWrite(receiptPath, receipt, 0o440); await chmod(receiptPath, 0o440); if (input.testPreparationCrashPoint === "after-receipt") throw Object.assign( new Error("Injected preparation process death after receipt."), { simulatedProcessDeath: true }, ); await durableCanonicalWrite(statePath, { ...stateAfterSmoke, activeTransaction: { ...activeTransaction, phase: "ready", readinessReceiptDigest: receiptDigest, }, }); return { receipt, receiptDigest, releaseDir }; } catch (error) { if (error?.simulatedProcessDeath === true) throw error; await makeWritable(partialDir).catch(() => undefined); await safeRemoveStage(root, partialName).catch(() => undefined); if ( (await currentTarget(root).catch(() => null)) !== `releases/${index.releaseId}/public` ) { await makeWritable(releaseDir).catch(() => undefined); await rm(releaseDir, { recursive: true, force: true }).catch( () => undefined, ); } const latest = await readControlState(root).catch(() => null); if ( latest?.activeTransaction?.transactionId === transactionId && latest.activeTransaction.phase !== "committing" ) { await durableCanonicalWrite(statePath, { ...latest, activeTransaction: null, }).catch(() => undefined); } throw error; } } export async function prepareRelease(input) { exactObject( input, ["root", "archivePath", "transactionId", "smokeCommand"], "production preparation request", ); const root = await assertReleaseRoot(input.root); const archive = archiveFromInbox(root, input.archivePath); return withExclusiveLock(root, () => prepareUnlocked({ ...input, root, archivePath: archive }), ); } export async function prepareReleaseForTest(input) { const root = await assertReleaseRoot(input.root); return input.lock === false ? prepareUnlocked({ ...input, root }) : withExclusiveLock(root, () => prepareUnlocked({ ...input, root })); } function validateReadiness(value) { const receipt = exactObject(value, readinessFields, "readiness receipt"); if (receipt.schema !== "portfolio-release-readiness-receipt/v1") throw new Error("Readiness receipt schema is invalid."); assertToken(receipt.transactionId, "readiness transaction ID"); assertDigest(receipt.manifestDigest, "readiness manifest digest"); assertToken(receipt.releaseId, "readiness release ID"); assertTimestamp(receipt.expiresAt, "readiness expiry"); assertDigest(receipt.latestLedgerHeadHash, "readiness ledger head"); assertDigest(receipt.packageDigest, "readiness package digest"); assertDigest(receipt.smokeEvidenceDigest, "smoke evidence digest"); if ( !Number.isSafeInteger(receipt.latestLedgerRevision) || receipt.latestLedgerRevision < 0 || !Number.isSafeInteger(receipt.expectedFence) || receipt.expectedFence < 1 ) throw new Error("Readiness receipt counters are invalid."); if (receipt.predecessorManifestDigest !== null) assertDigest( receipt.predecessorManifestDigest, "readiness predecessor manifest digest", ); return receipt; } async function signedManifestFromRelease(root, releaseId) { return join( root, "releases", releaseId, ".release/signed-release-manifest.json", ); } async function currentTarget(root) { try { return await readlink(join(root, "current")); } catch (error) { if (error?.code === "ENOENT") return null; throw error; } } async function finalizeActivation( root, state, tx, receipt, committedAt, recovered, ) { const newState = { ...state, currentManifestDigest: tx.manifestDigest, currentReleaseId: tx.releaseId, publisherFence: tx.expectedFence, activeTransaction: null, }; await mkdir(join(root, "receipts", "activation"), { recursive: true, mode: 0o750, }); const activationReceipt = { schema: "portfolio-release-activation-receipt/v1", transactionId: tx.transactionId, predecessorManifestDigest: receipt.predecessorManifestDigest, manifestDigest: tx.manifestDigest, releaseId: tx.releaseId, fence: tx.expectedFence, readinessReceiptDigest: canonicalJsonDigest( "portfolio-release-readiness-receipt/v1", receipt, ), committedAt, recoveredAfterPointerSwap: recovered, }; const path = join(root, "receipts", "activation", `${tx.transactionId}.json`); await durableCanonicalWrite(path, activationReceipt, 0o440); await chmod(path, 0o440); await durableCanonicalWrite(join(root, "control/state.json"), newState); return activationReceipt; } async function activateUnlocked(input) { const root = await assertReleaseRoot(input.root); const transactionId = assertToken(input.transactionId, "transaction ID"); const expectedCurrent = normalizeExpectedDigest( input.expectedCurrentManifestDigest, ); if (!Number.isSafeInteger(input.expectedFence) || input.expectedFence < 1) throw new Error("Expected fence is invalid."); let state = await readControlState(root); const tx = state.activeTransaction; if ( tx === null || tx.transactionId !== transactionId || !["ready", "committing"].includes(tx.phase) ) throw new Error("Named transaction is not ready for activation."); const receipt = validateReadiness( parseCanonicalJson( await readFile( join(root, "receipts", "readiness", `${transactionId}.json`), ), ), ); if ( tx.readinessReceiptDigest !== canonicalJsonDigest("portfolio-release-readiness-receipt/v1", receipt) ) throw new Error( "Readiness receipt digest does not match the protected transaction.", ); const target = `releases/${tx.releaseId}/public`; const predecessorTarget = state.currentReleaseId === null ? null : `releases/${state.currentReleaseId}/public`; const actualCurrent = await currentTarget(root); if ( state.currentManifestDigest !== expectedCurrent || tx.expectedFence !== input.expectedFence || state.publisherFence + 1 !== input.expectedFence || receipt.predecessorManifestDigest !== expectedCurrent || receipt.expectedFence !== input.expectedFence ) throw new Error("Activation predecessor or fence is stale."); if ( (tx.phase === "ready" && actualCurrent !== predecessorTarget) || (tx.phase === "committing" && actualCurrent !== predecessorTarget && actualCurrent !== target) ) { throw new Error( "Public pointer does not match protected activation state.", ); } const activationNow = nowFrom(input.testClock); if (Date.parse(activationNow) >= Date.parse(receipt.expiresAt)) throw new Error("Release expired before activation."); const verificationInput = { signedManifestPath: await signedManifestFromRelease(root, tx.releaseId), protectedState: protectedStateFrom(state), }; const verified = input.testOnlyVerification ? await verifySignedReleaseForTest({ ...verificationInput, ...input.testOnlyVerification, now: activationNow, }) : await verifySignedRelease({ ...verificationInput, authorityRoot: root }); if ( verified.admission.manifestDigest !== tx.manifestDigest || verified.admission.expectedFence !== tx.expectedFence || verified.admission.latestLedgerHeadHash !== receipt.latestLedgerHeadHash || verified.admission.latestLedgerRevision !== receipt.latestLedgerRevision ) { throw new Error("Activation signature admission does not match readiness."); } if (input.testBeforeCommit !== undefined) await input.testBeforeCommit(); state = await readControlState(root); if ( state.currentManifestDigest !== expectedCurrent || state.publisherFence + 1 !== input.expectedFence || state.activeTransaction?.transactionId !== transactionId || state.latestLedgerHeadHash !== receipt.latestLedgerHeadHash || state.latestLedgerRevision !== receipt.latestLedgerRevision || state.releaseTrustPolicyDigest !== verificationInput.protectedState.releaseTrustPolicyDigest ) { throw new Error("Protected state changed before activation commit."); } if (tx.phase === "committing" && actualCurrent === target) { return finalizeActivation(root, state, tx, receipt, activationNow, true); } await durableCanonicalWrite(join(root, "control/state.json"), { ...state, activeTransaction: { ...tx, phase: "committing" }, }); if (input.testBeforePointerSwap !== undefined) await input.testBeforePointerSwap(); const finalState = await readControlState(root); if ( finalState.activeTransaction?.transactionId !== transactionId || finalState.activeTransaction.phase !== "committing" || finalState.latestLedgerHeadHash !== receipt.latestLedgerHeadHash || finalState.latestLedgerRevision !== receipt.latestLedgerRevision || finalState.releaseTrustPolicyDigest !== verificationInput.protectedState.releaseTrustPolicyDigest || finalState.currentManifestDigest !== expectedCurrent || finalState.publisherFence + 1 !== input.expectedFence || (await currentTarget(root)) !== predecessorTarget ) { throw new Error("Protected authority changed before pointer publication."); } const publicationNow = nowFrom(input.testClock); if (Date.parse(publicationNow) >= Date.parse(receipt.expiresAt)) throw new Error("Release expired before pointer publication."); const nextPath = join(root, "current.next"); const nextInfo = await lstat(nextPath).catch((error) => error?.code === "ENOENT" ? null : Promise.reject(error), ); if (nextInfo !== null) { if (!nextInfo.isSymbolicLink()) throw new Error( "current.next is not a symlink and will not be overwritten.", ); await unlink(nextPath); } await symlink(target, nextPath); if (input.crashPoint === "before-swap") throw new Error("Injected crash before pointer swap."); await rename(nextPath, join(root, "current")); await syncDirectory(root); if (input.crashPoint === "after-swap") throw new Error("Injected crash after pointer swap."); return finalizeActivation( root, finalState, tx, receipt, publicationNow, false, ); } export async function activateRelease(input) { exactObject( input, ["root", "transactionId", "expectedCurrentManifestDigest", "expectedFence"], "production activation request", ); const root = await assertReleaseRoot(input.root); return withExclusiveLock(root, () => activateUnlocked({ ...input, root })); } export async function activateReleaseForTest(input) { const root = await assertReleaseRoot(input.root); return input.lock === false ? activateUnlocked({ ...input, root }) : withExclusiveLock(root, () => activateUnlocked({ ...input, root })); } export async function rollbackRelease(input) { exactObject( input, [ "root", "targetReleaseId", "archivePath", "transactionId", "smokeCommand", "expectedCurrentManifestDigest", "expectedFence", ], "production rollback request", ); const root = await assertReleaseRoot(input.root); const archivePath = archiveFromInbox(root, input.archivePath); return withExclusiveLock(root, async () => { await prepareUnlocked({ ...input, root, archivePath, rollbackTargetReleaseId: input.targetReleaseId, lock: false, }); return activateUnlocked({ ...input, root, lock: false }); }); } export async function rollbackReleaseForTest(input) { const root = await assertReleaseRoot(input.root); return withExclusiveLock(root, async () => { await prepareUnlocked({ ...input, root, rollbackTargetReleaseId: input.targetReleaseId, lock: false, }); return activateUnlocked({ ...input, root, lock: false }); }); } async function makeWritable(path) { const info = await lstat(path); if (info.isDirectory()) { await chmod(path, 0o700); for (const entry of await readdir(path)) await makeWritable(join(path, entry)); } else if (info.isFile()) await chmod(path, 0o600); else throw new Error("Orphan staging contains a non-file entry."); } function archiveFromInbox(root, archivePath) { const archive = resolve(archivePath); const inbox = join(root, "inbox"); const rel = relative(inbox, archive); if (rel === "" || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw new Error("Production archive must come from the upload-only inbox."); return archive; } export async function cleanupOrphanStages(rootPath) { const root = await assertReleaseRoot(rootPath); const staging = join(root, "staging"); await mkdir(staging, { recursive: true, mode: 0o750 }); const state = await readControlState(root); const activePrefix = state.activeTransaction === null ? null : `${state.activeTransaction.transactionId}.`; const removed = []; for (const entry of await readdir(staging, { withFileTypes: true })) { if ( !entry.isDirectory() || entry.isSymbolicLink() || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.partial$/u.test(entry.name) ) { throw new Error("Unexpected staging entry blocks orphan cleanup."); } if (activePrefix !== null && entry.name.startsWith(activePrefix)) continue; const path = join(staging, entry.name); await makeWritable(path); await safeRemoveStage(root, entry.name); removed.push(entry.name); } return removed; }