fix: harden additive asset publication
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
const idPattern = /^[a-z][a-z0-9-]{0,63}$/u;
|
||||
const fileNamePattern = /^[a-z0-9][a-z0-9-]*\.(avif|jpg|png|svg|webp|mp4|webm|opus|mp3)$/u;
|
||||
const managedFileNamePattern = /^(?:project|social)-[a-z0-9][a-z0-9-]*\.(?:avif|jpg|png|svg|webp|mp4|webm)$/u;
|
||||
const sha256Pattern = /^[a-f0-9]{64}$/u;
|
||||
|
||||
const assertSafeRecord = (asset) => {
|
||||
@@ -21,7 +22,8 @@ export const mergeAssetRecords = (baseAssets, generatedAssets, managedIds) => {
|
||||
throw new Error('ASSET-MERGE-RECORD-001');
|
||||
}
|
||||
}
|
||||
const retained = baseAssets.filter(({ id }) => !managedIds.has(id));
|
||||
const retained = baseAssets.filter(({ id, fileName }) =>
|
||||
!managedIds.has(id) || !managedFileNamePattern.test(fileName));
|
||||
const combined = [...retained, ...generatedAssets];
|
||||
const ids = new Set();
|
||||
const names = new Set();
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash } from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import { copyFile, lstat, mkdir, open, readFile, readdir, rm, unlink, writeFile } from 'node:fs/promises';
|
||||
import { basename, dirname, extname, join, resolve } from 'node:path';
|
||||
import { lstat, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, writeFile } from 'node:fs/promises';
|
||||
import { basename, dirname, extname, join, resolve, sep } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import sharp from 'sharp';
|
||||
import { fileTypeFromFile } from 'file-type';
|
||||
@@ -15,6 +15,44 @@ import { encodeVideoDerivatives, extractVideoFrames, probeVideo } from './lib/vi
|
||||
|
||||
const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const fail = (rule) => { throw new Error(rule); };
|
||||
const isErrno = (error, code) => typeof error === 'object' && error !== null && 'code' in error && error.code === code;
|
||||
|
||||
const canonicalizePathRole = async (path) => {
|
||||
let cursor = resolve(path);
|
||||
const suffix = [];
|
||||
while (true) {
|
||||
try {
|
||||
return resolve(await realpath(cursor), ...suffix);
|
||||
} catch (error) {
|
||||
if (!isErrno(error, 'ENOENT')) fail('ASSET-PATH-001');
|
||||
const parent = dirname(cursor);
|
||||
if (parent === cursor) fail('ASSET-PATH-001');
|
||||
suffix.unshift(basename(cursor));
|
||||
cursor = parent;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const pathsOverlap = (left, right) =>
|
||||
left === right || left.startsWith(`${right}${sep}`) || right.startsWith(`${left}${sep}`);
|
||||
|
||||
const validatePathRoles = async (options) => {
|
||||
const roleNames = [
|
||||
'sourceRoot', 'mappingPath', 'denylistPath', 'recipesPath', 'stagingRoot', 'reportDir',
|
||||
...(options.approvalRegisterPath === undefined ? [] : ['approvalRegisterPath', 'publishRoot', 'manifestPath']),
|
||||
];
|
||||
const roles = Object.fromEntries(await Promise.all(roleNames.map(async (name) => [
|
||||
name,
|
||||
await canonicalizePathRole(options[name]),
|
||||
])));
|
||||
for (const resetRole of ['stagingRoot', 'reportDir']) {
|
||||
if (!roles[resetRole].startsWith(`/tmp${sep}`)) fail('ASSET-PATH-001');
|
||||
for (const otherRole of roleNames) {
|
||||
if (resetRole !== otherRole && pathsOverlap(roles[resetRole], roles[otherRole])) fail('ASSET-PATH-001');
|
||||
}
|
||||
}
|
||||
return roles;
|
||||
};
|
||||
|
||||
const assertSafeOutputRoot = (path, rule) => {
|
||||
const resolved = resolve(path);
|
||||
@@ -210,8 +248,12 @@ const validateManifestRecord = (asset) => {
|
||||
}
|
||||
};
|
||||
|
||||
const loadManifestAssets = async (manifestPath) => {
|
||||
const manifest = await readJson(manifestPath, 'ASSET-MANIFEST-001');
|
||||
const fileIdentity = (details) => ({ dev: details.dev, ino: details.ino });
|
||||
const sameIdentity = (left, right) => left.dev === right.dev && left.ino === right.ino;
|
||||
|
||||
const manifestAssetsFromBytes = (bytes) => {
|
||||
let manifest;
|
||||
try { manifest = JSON.parse(bytes.toString('utf8')); } catch { fail('ASSET-MANIFEST-001'); }
|
||||
if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest) ||
|
||||
Object.keys(manifest).length !== 1 || !Array.isArray(manifest.assets)) fail('ASSET-MANIFEST-001');
|
||||
for (const asset of manifest.assets) validateManifestRecord(asset);
|
||||
@@ -231,7 +273,7 @@ const readVerifiedRegularFile = async (path, expectedHash, rule) => {
|
||||
if (!details.isFile() || details.nlink !== 1) fail(rule);
|
||||
const bytes = await handle.readFile();
|
||||
if (createHash('sha256').update(bytes).digest('hex') !== expectedHash) fail('ASSET-HASH-001');
|
||||
return bytes;
|
||||
return { bytes, identity: fileIdentity(details) };
|
||||
} catch (error) {
|
||||
if (error instanceof Error && [rule, 'ASSET-HASH-001'].includes(error.message)) throw error;
|
||||
fail(rule);
|
||||
@@ -240,61 +282,167 @@ const readVerifiedRegularFile = async (path, expectedHash, rule) => {
|
||||
}
|
||||
};
|
||||
|
||||
const verifyCurrentPublication = async (publishRoot, assets) => {
|
||||
const safePublishRoot = assertSafeOutputRoot(publishRoot, 'ASSET-PUBLISH-001');
|
||||
const readManifestSnapshot = async (manifestPath) => {
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(manifestPath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
||||
const details = await handle.stat();
|
||||
if (!details.isFile() || details.nlink !== 1) fail('ASSET-MANIFEST-001');
|
||||
const bytes = await handle.readFile();
|
||||
return { assets: manifestAssetsFromBytes(bytes), bytes, identity: fileIdentity(details) };
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'ASSET-MANIFEST-001') throw error;
|
||||
fail('ASSET-MANIFEST-001');
|
||||
} finally {
|
||||
await handle?.close();
|
||||
}
|
||||
};
|
||||
|
||||
const verifyPublicationDirectory = async (publishRoot, assets) => {
|
||||
let rootStats;
|
||||
try { rootStats = await lstat(safePublishRoot); } catch { fail('ASSET-EXACT-001'); }
|
||||
try { rootStats = await lstat(publishRoot); } catch { fail('ASSET-EXACT-001'); }
|
||||
if (rootStats.isSymbolicLink() || !rootStats.isDirectory()) fail('ASSET-EXACT-001');
|
||||
const expectedNames = new Set(assets.map(({ fileName }) => fileName));
|
||||
const entries = await readdir(safePublishRoot, { withFileTypes: true });
|
||||
const entries = await readdir(publishRoot, { withFileTypes: true });
|
||||
if (entries.length !== expectedNames.size) fail('ASSET-EXACT-001');
|
||||
for (const entry of entries) {
|
||||
if (!expectedNames.has(entry.name) || entry.isSymbolicLink() || !entry.isFile()) fail('ASSET-EXACT-001');
|
||||
}
|
||||
const files = new Map();
|
||||
for (const asset of assets) {
|
||||
await readVerifiedRegularFile(join(safePublishRoot, asset.fileName), asset.sha256, 'ASSET-EXACT-001');
|
||||
const snapshot = await readVerifiedRegularFile(join(publishRoot, asset.fileName), asset.sha256, 'ASSET-EXACT-001');
|
||||
files.set(asset.fileName, snapshot);
|
||||
}
|
||||
return safePublishRoot;
|
||||
return { rootIdentity: fileIdentity(rootStats), files };
|
||||
};
|
||||
|
||||
const publish = async ({ candidates, recipes, stagingRoot, publishRoot, manifestPath }) => {
|
||||
const verifyCurrentPublication = async (publishRoot, assets) => {
|
||||
const safePublishRoot = assertSafeOutputRoot(publishRoot, 'ASSET-PUBLISH-001');
|
||||
return { safePublishRoot, ...await verifyPublicationDirectory(safePublishRoot, assets) };
|
||||
};
|
||||
|
||||
const managedCoordinatesForRecipes = (recipes) => recipes.flatMap((recipe) => {
|
||||
if (recipe.kind === 'image') return [{ id: recipe.id, fileName: recipe.output }];
|
||||
if (recipe.kind !== 'video' || !recipe.output.endsWith('.webm')) fail('ASSET-OWNERSHIP-001');
|
||||
const base = recipe.output.slice(0, -'.webm'.length);
|
||||
return [
|
||||
{ id: recipe.id, fileName: `${base}.webm` },
|
||||
{ id: `${recipe.id}-mp4`, fileName: `${base}.mp4` },
|
||||
{ id: `${recipe.id}-poster`, fileName: `${base}-poster.webp` },
|
||||
];
|
||||
});
|
||||
|
||||
const trustedManagedIds = (recipes, candidates) => {
|
||||
const expected = managedCoordinatesForRecipes(recipes);
|
||||
const expectedCoordinates = expected.map(({ id, fileName }) => `${id}\u0000${fileName}`).sort();
|
||||
const actualCoordinates = candidates.map(({ id, fileName }) => `${id}\u0000${fileName}`).sort();
|
||||
if (JSON.stringify(actualCoordinates) !== JSON.stringify(expectedCoordinates)) fail('ASSET-OWNERSHIP-001');
|
||||
return new Set(expected.map(({ id }) => id));
|
||||
};
|
||||
|
||||
export const publishPreparedAssets = async ({ candidates, recipes, stagingRoot, publishRoot, manifestPath }) => {
|
||||
if (candidates.some((item) => /pending|requires confirmation/iu.test(`${item.rightsHolder} ${item.license}`))) {
|
||||
fail('ASSET-RIGHTS-001');
|
||||
}
|
||||
const baseAssets = await loadManifestAssets(manifestPath);
|
||||
const safePublishRoot = await verifyCurrentPublication(publishRoot, baseAssets);
|
||||
const managedIds = new Set(recipes.map(({ id }) => id));
|
||||
const manifestSnapshot = await readManifestSnapshot(manifestPath);
|
||||
const baseAssets = manifestSnapshot.assets;
|
||||
const currentPublication = await verifyCurrentPublication(publishRoot, baseAssets);
|
||||
const { safePublishRoot } = currentPublication;
|
||||
const managedIds = trustedManagedIds(recipes, candidates);
|
||||
const mergedAssets = mergeAssetRecords(baseAssets, candidates, managedIds);
|
||||
const candidateSnapshots = new Map();
|
||||
for (const item of candidates) {
|
||||
validateManifestRecord(item);
|
||||
await readVerifiedRegularFile(join(stagingRoot, item.fileName), item.sha256, 'ASSET-EXACT-001');
|
||||
const path = join(stagingRoot, item.fileName);
|
||||
candidateSnapshots.set(item.fileName, {
|
||||
path,
|
||||
expectedHash: item.sha256,
|
||||
...await readVerifiedRegularFile(path, item.sha256, 'ASSET-EXACT-001'),
|
||||
});
|
||||
}
|
||||
|
||||
const mergedNames = new Set(mergedAssets.map(({ fileName }) => fileName));
|
||||
const obsoleteNames = baseAssets
|
||||
.filter(({ id, fileName }) => managedIds.has(id) && !mergedNames.has(fileName))
|
||||
.map(({ fileName }) => fileName);
|
||||
for (const item of candidates) await copyFile(join(stagingRoot, item.fileName), join(safePublishRoot, item.fileName));
|
||||
for (const item of candidates) {
|
||||
await readVerifiedRegularFile(join(safePublishRoot, item.fileName), item.sha256, 'ASSET-EXACT-001');
|
||||
let publicationTransactionRoot;
|
||||
let manifestTransactionRoot;
|
||||
let preserveRecoveryState = false;
|
||||
try {
|
||||
publicationTransactionRoot = await mkdtemp(join(dirname(safePublishRoot), '.asset-publication-'));
|
||||
const nextPublication = join(publicationTransactionRoot, 'next');
|
||||
const previousPublication = join(publicationTransactionRoot, 'previous');
|
||||
await mkdir(nextPublication);
|
||||
for (const asset of mergedAssets) {
|
||||
const snapshot = candidateSnapshots.get(asset.fileName) ?? currentPublication.files.get(asset.fileName);
|
||||
if (snapshot === undefined) fail('ASSET-PUBLISH-001');
|
||||
await writeFile(join(nextPublication, asset.fileName), snapshot.bytes, { flag: 'wx', mode: 0o644 });
|
||||
}
|
||||
await verifyPublicationDirectory(nextPublication, mergedAssets);
|
||||
|
||||
manifestTransactionRoot = await mkdtemp(join(dirname(manifestPath), '.asset-manifest-'));
|
||||
const nextManifest = join(manifestTransactionRoot, 'next-manifest.json');
|
||||
const previousManifest = join(manifestTransactionRoot, 'previous-manifest.json');
|
||||
await writeFile(nextManifest, `${JSON.stringify({ assets: mergedAssets }, null, 2)}\n`, { flag: 'wx', mode: 0o644 });
|
||||
|
||||
for (const snapshot of candidateSnapshots.values()) {
|
||||
const current = await readVerifiedRegularFile(snapshot.path, snapshot.expectedHash, 'ASSET-EXACT-001');
|
||||
if (!sameIdentity(current.identity, snapshot.identity)) fail('ASSET-EXACT-001');
|
||||
}
|
||||
const protectedPublication = await verifyCurrentPublication(safePublishRoot, baseAssets);
|
||||
if (!sameIdentity(protectedPublication.rootIdentity, currentPublication.rootIdentity)) fail('ASSET-EXACT-001');
|
||||
const protectedManifest = await readManifestSnapshot(manifestPath);
|
||||
if (!sameIdentity(protectedManifest.identity, manifestSnapshot.identity) ||
|
||||
!protectedManifest.bytes.equals(manifestSnapshot.bytes)) fail('ASSET-MANIFEST-001');
|
||||
|
||||
let publicationBackedUp = false;
|
||||
let publicationInstalled = false;
|
||||
let manifestBackedUp = false;
|
||||
let manifestInstalled = false;
|
||||
try {
|
||||
await rename(safePublishRoot, previousPublication);
|
||||
publicationBackedUp = true;
|
||||
await rename(nextPublication, safePublishRoot);
|
||||
publicationInstalled = true;
|
||||
await rename(manifestPath, previousManifest);
|
||||
manifestBackedUp = true;
|
||||
await rename(nextManifest, manifestPath);
|
||||
manifestInstalled = true;
|
||||
const installedManifest = await readManifestSnapshot(manifestPath);
|
||||
if (JSON.stringify(installedManifest.assets) !== JSON.stringify(mergedAssets)) fail('ASSET-PUBLISH-001');
|
||||
await verifyCurrentPublication(safePublishRoot, mergedAssets);
|
||||
} catch {
|
||||
try {
|
||||
if (manifestInstalled) await rename(manifestPath, nextManifest);
|
||||
if (manifestBackedUp) await rename(previousManifest, manifestPath);
|
||||
if (publicationInstalled) await rename(safePublishRoot, nextPublication);
|
||||
if (publicationBackedUp) await rename(previousPublication, safePublishRoot);
|
||||
} catch {
|
||||
preserveRecoveryState = true;
|
||||
fail('ASSET-PUBLISH-ROLLBACK-001');
|
||||
}
|
||||
fail('ASSET-PUBLISH-001');
|
||||
}
|
||||
} finally {
|
||||
if (!preserveRecoveryState) {
|
||||
if (manifestTransactionRoot !== undefined) await rm(manifestTransactionRoot, { recursive: true, force: true });
|
||||
if (publicationTransactionRoot !== undefined) await rm(publicationTransactionRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
for (const fileName of obsoleteNames) await unlink(join(safePublishRoot, fileName));
|
||||
await verifyCurrentPublication(safePublishRoot, mergedAssets);
|
||||
await mkdir(dirname(manifestPath), { recursive: true });
|
||||
await writeFile(manifestPath, `${JSON.stringify({ assets: mergedAssets }, null, 2)}\n`, 'utf8');
|
||||
};
|
||||
|
||||
export const prepareAssets = async (options) => {
|
||||
const required = ['sourceRoot', 'mappingPath', 'denylistPath', 'recipesPath', 'stagingRoot', 'reportDir'];
|
||||
if (!options || required.some((key) => typeof options[key] !== 'string')) fail('ASSET-ARGUMENT-001');
|
||||
const recipes = await loadRecipes(options.recipesPath);
|
||||
const mapping = await loadPrivateMapping(options.mappingPath, recipes);
|
||||
const denylist = await loadDenylist(options.denylistPath);
|
||||
const stagingRoot = await resetDirectory(options.stagingRoot, 'ASSET-STAGING-001');
|
||||
const reportDir = await resetDirectory(options.reportDir, 'ASSET-REPORT-001');
|
||||
if (options.approvalRegisterPath !== undefined &&
|
||||
!['approvalRegisterPath', 'publishRoot', 'manifestPath'].every((key) => typeof options[key] === 'string')) {
|
||||
fail('ASSET-ARGUMENT-001');
|
||||
}
|
||||
const paths = await validatePathRoles(options);
|
||||
const recipes = await loadRecipes(paths.recipesPath);
|
||||
const mapping = await loadPrivateMapping(paths.mappingPath, recipes);
|
||||
const denylist = await loadDenylist(paths.denylistPath);
|
||||
const stagingRoot = await resetDirectory(paths.stagingRoot, 'ASSET-STAGING-001');
|
||||
const reportDir = await resetDirectory(paths.reportDir, 'ASSET-REPORT-001');
|
||||
const candidates = [];
|
||||
for (const recipe of recipes) {
|
||||
const source = await resolvePrivateSource(options.sourceRoot, mapping[recipe.id]);
|
||||
const source = await resolvePrivateSource(paths.sourceRoot, mapping[recipe.id]);
|
||||
const prepared = recipe.kind === 'image'
|
||||
? await prepareImage({ source, recipe, stagingRoot, denylist })
|
||||
: await prepareVideo({ source, recipe, stagingRoot, reportDir, denylist });
|
||||
@@ -310,11 +458,10 @@ export const prepareAssets = async (options) => {
|
||||
await createContactSheet(finalCandidates, stagingRoot, reportDir);
|
||||
let approvalStatus = 'pending';
|
||||
if (options.approvalRegisterPath !== undefined) {
|
||||
if (typeof options.publishRoot !== 'string' || typeof options.manifestPath !== 'string') fail('ASSET-ARGUMENT-001');
|
||||
await approvalsMatch(options.approvalRegisterPath, approval);
|
||||
await publish({
|
||||
await approvalsMatch(paths.approvalRegisterPath, approval);
|
||||
await publishPreparedAssets({
|
||||
candidates: finalCandidates, recipes, stagingRoot,
|
||||
publishRoot: options.publishRoot, manifestPath: options.manifestPath,
|
||||
publishRoot: paths.publishRoot, manifestPath: paths.manifestPath,
|
||||
});
|
||||
approvalStatus = 'approved';
|
||||
}
|
||||
@@ -323,8 +470,8 @@ export const prepareAssets = async (options) => {
|
||||
|
||||
export const verifyPreparedAssets = async ({ publishRoot, manifestPath, denylistPath }) => {
|
||||
if (![publishRoot, manifestPath, denylistPath].every((value) => typeof value === 'string')) fail('ASSET-ARGUMENT-001');
|
||||
const assets = await loadManifestAssets(manifestPath);
|
||||
const safePublishRoot = await verifyCurrentPublication(publishRoot, assets);
|
||||
const { assets } = await readManifestSnapshot(manifestPath);
|
||||
const { safePublishRoot } = await verifyCurrentPublication(publishRoot, assets);
|
||||
const denylist = await loadDenylist(denylistPath);
|
||||
for (const item of assets) {
|
||||
const path = join(safePublishRoot, item.fileName);
|
||||
|
||||
Reference in New Issue
Block a user