372 lines
17 KiB
JavaScript
372 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
import { createHash } from 'node:crypto';
|
|
import { constants } from 'node:fs';
|
|
import { copyFile, lstat, mkdir, open, readFile, readdir, rm, unlink, writeFile } from 'node:fs/promises';
|
|
import { basename, dirname, extname, join, resolve } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import sharp from 'sharp';
|
|
import { fileTypeFromFile } from 'file-type';
|
|
|
|
import { hashFile } from './lib/hash-file.mjs';
|
|
import { loadPrivateMapping, loadRecipes, readJson, resolvePrivateSource } from './lib/asset-recipes.mjs';
|
|
import { mergeAssetRecords } from './lib/merge-assets.mjs';
|
|
import { loadDenylist, scanBinary, scanImage } from './lib/privacy-scan.mjs';
|
|
import { encodeVideoDerivatives, extractVideoFrames, probeVideo } from './lib/video-frames.mjs';
|
|
|
|
const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const fail = (rule) => { throw new Error(rule); };
|
|
|
|
const assertSafeOutputRoot = (path, rule) => {
|
|
const resolved = resolve(path);
|
|
const permittedRepositoryOutput = join(repositoryRoot, 'inputs', 'prepared-assets');
|
|
if (resolved !== permittedRepositoryOutput && !resolved.startsWith('/tmp/')) fail(rule);
|
|
return resolved;
|
|
};
|
|
|
|
const resetDirectory = async (path, rule) => {
|
|
const safe = assertSafeOutputRoot(path, rule);
|
|
await rm(safe, { recursive: true, force: true });
|
|
await mkdir(safe, { recursive: true });
|
|
return safe;
|
|
};
|
|
|
|
const sorted = (items) => [...items].sort((left, right) => left.fileName.localeCompare(right.fileName, 'en-US'));
|
|
|
|
const candidate = async ({ id, fileName, path, mediaType, width, height, durationMs, recipe }) => ({
|
|
id,
|
|
fileName,
|
|
sha256: await hashFile(path),
|
|
mediaType,
|
|
width,
|
|
height,
|
|
durationMs,
|
|
rightsHolder: recipe.rightsHolder,
|
|
license: recipe.license,
|
|
attribution: recipe.attribution,
|
|
});
|
|
|
|
const encodeImage = async (source, output, recipe) => {
|
|
let detected;
|
|
try { detected = await fileTypeFromFile(source); } catch { fail('ASSET-TYPE-001'); }
|
|
if (!detected || !['image/jpeg', 'image/png', 'image/webp'].includes(detected.mime)) fail('ASSET-TYPE-001');
|
|
try {
|
|
let pipeline = sharp(source, { failOn: 'error', limitInputPixels: 50_000_000, sequentialRead: true })
|
|
.rotate()
|
|
.resize({ width: recipe.maxWidth, withoutEnlargement: true, fit: 'inside', kernel: sharp.kernel.lanczos3 });
|
|
pipeline = recipe.output.endsWith('.png')
|
|
? pipeline.png({ compressionLevel: 9, adaptiveFiltering: false, palette: false })
|
|
: pipeline.webp({ lossless: true, effort: 6, smartSubsample: false });
|
|
await pipeline.toFile(output);
|
|
} catch {
|
|
fail('ASSET-DECODE-001');
|
|
}
|
|
const details = await scanImage(output, [], { runOcr: false });
|
|
return details;
|
|
};
|
|
|
|
const prepareImage = async ({ source, recipe, stagingRoot, denylist }) => {
|
|
const path = join(stagingRoot, recipe.output);
|
|
const details = await encodeImage(source, path, recipe);
|
|
await scanImage(path, denylist, { runOcr: true });
|
|
return [await candidate({
|
|
id: recipe.id, fileName: recipe.output, path, mediaType: details.mediaType,
|
|
width: details.width, height: details.height, durationMs: null, recipe,
|
|
})];
|
|
};
|
|
|
|
const prepareVideo = async ({ source, recipe, stagingRoot, reportDir, denylist }) => {
|
|
const sourceProbe = await probeVideo(source);
|
|
if (recipe.startSeconds + recipe.durationSeconds > sourceProbe.duration + 0.05) fail('ASSET-VIDEO-RANGE-001');
|
|
const base = recipe.output.slice(0, -'.webm'.length);
|
|
const webmPath = join(stagingRoot, `${base}.webm`);
|
|
const mp4Path = join(stagingRoot, `${base}.mp4`);
|
|
const posterPath = join(stagingRoot, `${base}-poster.webp`);
|
|
await encodeVideoDerivatives({
|
|
source, webmPath, mp4Path, posterPath,
|
|
startSeconds: recipe.startSeconds, durationSeconds: recipe.durationSeconds, maxWidth: recipe.maxWidth,
|
|
});
|
|
const webmProbe = await probeVideo(webmPath, { allowAudio: false });
|
|
const mp4Probe = await probeVideo(mp4Path, { allowAudio: false });
|
|
const frames = await extractVideoFrames({
|
|
source: webmPath,
|
|
durationSeconds: Math.min(webmProbe.duration, recipe.durationSeconds),
|
|
outputDir: join(reportDir, 'video-frames'),
|
|
prefix: recipe.id,
|
|
});
|
|
await scanBinary(webmPath, denylist);
|
|
await scanBinary(mp4Path, denylist);
|
|
await scanImage(posterPath, denylist, { runOcr: true });
|
|
for (const frame of frames) await scanImage(frame, denylist, { runOcr: true });
|
|
const poster = await sharp(posterPath).metadata();
|
|
return [
|
|
await candidate({
|
|
id: recipe.id, fileName: `${base}.webm`, path: webmPath, mediaType: 'video/webm',
|
|
width: webmProbe.width, height: webmProbe.height, durationMs: Math.round(webmProbe.duration * 1000), recipe,
|
|
}),
|
|
await candidate({
|
|
id: `${recipe.id}-mp4`, fileName: `${base}.mp4`, path: mp4Path, mediaType: 'video/mp4',
|
|
width: mp4Probe.width, height: mp4Probe.height, durationMs: Math.round(mp4Probe.duration * 1000), recipe,
|
|
}),
|
|
await candidate({
|
|
id: `${recipe.id}-poster`, fileName: `${base}-poster.webp`, path: posterPath, mediaType: 'image/webp',
|
|
width: poster.width, height: poster.height, durationMs: null, recipe,
|
|
}),
|
|
];
|
|
};
|
|
|
|
const createContactSheet = async (candidates, stagingRoot, reportDir) => {
|
|
const images = candidates.filter(({ mediaType }) => mediaType.startsWith('image/')).map((image) => ({
|
|
id: image.id,
|
|
path: join(stagingRoot, image.fileName),
|
|
}));
|
|
let frameNames = [];
|
|
try { frameNames = (await readdir(join(reportDir, 'video-frames'))).filter((name) => name.endsWith('.png')).sort(); } catch {}
|
|
images.push(...frameNames.map((name, index) => ({
|
|
id: `private-game-loop-frame-${String(index + 1).padStart(2, '0')}`,
|
|
path: join(reportDir, 'video-frames', name),
|
|
})));
|
|
if (images.length === 0) return;
|
|
const imageWidth = 1100;
|
|
const width = 1200;
|
|
const composites = [];
|
|
let y = 30;
|
|
for (const image of images) {
|
|
const thumb = await sharp(image.path)
|
|
.resize({ width: imageWidth, height: 1400, fit: 'inside', withoutEnlargement: false })
|
|
.png()
|
|
.toBuffer();
|
|
const metadata = await sharp(thumb).metadata();
|
|
const left = Math.round((width - (metadata.width ?? imageWidth)) / 2);
|
|
composites.push({ input: thumb, left, top: y });
|
|
const safeLabel = image.id.replace(/[^a-z0-9-]/gu, '');
|
|
const label = Buffer.from(`<svg width="1100" height="54" xmlns="http://www.w3.org/2000/svg"><rect width="1100" height="54" fill="#11131a"/><text x="16" y="36" font-size="24" font-family="sans-serif" fill="#f4f2f7">${safeLabel}</text></svg>`);
|
|
y += metadata.height ?? 0;
|
|
composites.push({ input: label, left: 50, top: y + 8 });
|
|
y += 86;
|
|
}
|
|
await sharp({ create: { width, height: y + 20, channels: 3, background: '#070910' } })
|
|
.composite(composites)
|
|
.webp({ lossless: true, effort: 6 })
|
|
.toFile(join(reportDir, 'contact-sheet.webp'));
|
|
};
|
|
|
|
const approvalProjection = (recipes, candidates) => ({
|
|
schema: 'portfolio-asset-approvals/v1',
|
|
assets: recipes.map(({ id }) => ({
|
|
id,
|
|
files: sorted(candidates.filter((item) => item.id === id || item.id.startsWith(`${id}-`)))
|
|
.map(({ fileName, sha256 }) => ({ fileName, sha256 })),
|
|
})).sort((left, right) => left.id.localeCompare(right.id, 'en-US')),
|
|
});
|
|
|
|
const approvalsMatch = async (path, expected) => {
|
|
const actual = await readJson(path, 'ASSET-APPROVAL-001');
|
|
if (JSON.stringify(actual) !== JSON.stringify(expected)) fail('ASSET-APPROVAL-002');
|
|
};
|
|
|
|
const manifestKeys = [
|
|
'id', 'fileName', 'sha256', 'mediaType', 'width', 'height', 'durationMs',
|
|
'rightsHolder', 'license', 'attribution',
|
|
];
|
|
const mediaExtension = new Map([
|
|
['image/avif', 'avif'], ['image/jpeg', 'jpg'], ['image/png', 'png'], ['image/svg+xml', 'svg'],
|
|
['image/webp', 'webp'], ['video/mp4', 'mp4'], ['video/webm', 'webm'],
|
|
['audio/ogg', 'opus'], ['audio/mpeg', 'mp3'],
|
|
]);
|
|
const isPositiveIntegerOrNull = (value) => value === null || (Number.isInteger(value) && value > 0);
|
|
const hasUnpairedSurrogate = (value) => {
|
|
for (let index = 0; index < value.length; index += 1) {
|
|
const codeUnit = value.charCodeAt(index);
|
|
if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) {
|
|
const next = value.charCodeAt(index + 1);
|
|
if (next < 0xdc00 || next > 0xdfff) return true;
|
|
index += 1;
|
|
} else if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
const isVisibleString = (value) => typeof value === 'string' && value.trim().length > 0 &&
|
|
value === value.normalize('NFC') && !hasUnpairedSurrogate(value) && !/[\u0000-\u001f\u007f]/u.test(value);
|
|
|
|
const validateManifestRecord = (asset) => {
|
|
if (!asset || typeof asset !== 'object' || Array.isArray(asset) ||
|
|
Object.keys(asset).length !== manifestKeys.length || manifestKeys.some((key) => !(key in asset)) ||
|
|
typeof asset.id !== 'string' || typeof asset.fileName !== 'string' || typeof asset.sha256 !== 'string' ||
|
|
!mediaExtension.has(asset.mediaType) || !asset.fileName.endsWith(`.${mediaExtension.get(asset.mediaType)}`) ||
|
|
!isPositiveIntegerOrNull(asset.width) || !isPositiveIntegerOrNull(asset.height) ||
|
|
!isPositiveIntegerOrNull(asset.durationMs) ||
|
|
!isVisibleString(asset.rightsHolder) || !isVisibleString(asset.license) || !isVisibleString(asset.attribution)) {
|
|
fail('ASSET-MANIFEST-001');
|
|
}
|
|
const isImage = asset.mediaType.startsWith('image/');
|
|
const isVideo = asset.mediaType.startsWith('video/');
|
|
const isAudio = asset.mediaType.startsWith('audio/');
|
|
if ((isImage && (asset.width === null || asset.height === null || asset.durationMs !== null)) ||
|
|
(isVideo && (asset.width === null || asset.height === null || asset.durationMs === null)) ||
|
|
(isAudio && (asset.width !== null || asset.height !== null || asset.durationMs === null))) {
|
|
fail('ASSET-MANIFEST-001');
|
|
}
|
|
};
|
|
|
|
const loadManifestAssets = async (manifestPath) => {
|
|
const manifest = await readJson(manifestPath, 'ASSET-MANIFEST-001');
|
|
if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest) ||
|
|
Object.keys(manifest).length !== 1 || !Array.isArray(manifest.assets)) fail('ASSET-MANIFEST-001');
|
|
for (const asset of manifest.assets) validateManifestRecord(asset);
|
|
try {
|
|
mergeAssetRecords(manifest.assets, [], new Set());
|
|
} catch {
|
|
fail('ASSET-MANIFEST-001');
|
|
}
|
|
return manifest.assets;
|
|
};
|
|
|
|
const readVerifiedRegularFile = async (path, expectedHash, rule) => {
|
|
let handle;
|
|
try {
|
|
handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
const details = await handle.stat();
|
|
if (!details.isFile() || details.nlink !== 1) fail(rule);
|
|
const bytes = await handle.readFile();
|
|
if (createHash('sha256').update(bytes).digest('hex') !== expectedHash) fail('ASSET-HASH-001');
|
|
return bytes;
|
|
} catch (error) {
|
|
if (error instanceof Error && [rule, 'ASSET-HASH-001'].includes(error.message)) throw error;
|
|
fail(rule);
|
|
} finally {
|
|
await handle?.close();
|
|
}
|
|
};
|
|
|
|
const verifyCurrentPublication = async (publishRoot, assets) => {
|
|
const safePublishRoot = assertSafeOutputRoot(publishRoot, 'ASSET-PUBLISH-001');
|
|
let rootStats;
|
|
try { rootStats = await lstat(safePublishRoot); } catch { fail('ASSET-EXACT-001'); }
|
|
if (rootStats.isSymbolicLink() || !rootStats.isDirectory()) fail('ASSET-EXACT-001');
|
|
const expectedNames = new Set(assets.map(({ fileName }) => fileName));
|
|
const entries = await readdir(safePublishRoot, { withFileTypes: true });
|
|
if (entries.length !== expectedNames.size) fail('ASSET-EXACT-001');
|
|
for (const entry of entries) {
|
|
if (!expectedNames.has(entry.name) || entry.isSymbolicLink() || !entry.isFile()) fail('ASSET-EXACT-001');
|
|
}
|
|
for (const asset of assets) {
|
|
await readVerifiedRegularFile(join(safePublishRoot, asset.fileName), asset.sha256, 'ASSET-EXACT-001');
|
|
}
|
|
return safePublishRoot;
|
|
};
|
|
|
|
const publish = async ({ candidates, recipes, stagingRoot, publishRoot, manifestPath }) => {
|
|
if (candidates.some((item) => /pending|requires confirmation/iu.test(`${item.rightsHolder} ${item.license}`))) {
|
|
fail('ASSET-RIGHTS-001');
|
|
}
|
|
const baseAssets = await loadManifestAssets(manifestPath);
|
|
const safePublishRoot = await verifyCurrentPublication(publishRoot, baseAssets);
|
|
const managedIds = new Set(recipes.map(({ id }) => id));
|
|
const mergedAssets = mergeAssetRecords(baseAssets, candidates, managedIds);
|
|
for (const item of candidates) {
|
|
validateManifestRecord(item);
|
|
await readVerifiedRegularFile(join(stagingRoot, item.fileName), item.sha256, 'ASSET-EXACT-001');
|
|
}
|
|
|
|
const mergedNames = new Set(mergedAssets.map(({ fileName }) => fileName));
|
|
const obsoleteNames = baseAssets
|
|
.filter(({ id, fileName }) => managedIds.has(id) && !mergedNames.has(fileName))
|
|
.map(({ fileName }) => fileName);
|
|
for (const item of candidates) await copyFile(join(stagingRoot, item.fileName), join(safePublishRoot, item.fileName));
|
|
for (const item of candidates) {
|
|
await readVerifiedRegularFile(join(safePublishRoot, item.fileName), item.sha256, 'ASSET-EXACT-001');
|
|
}
|
|
for (const fileName of obsoleteNames) await unlink(join(safePublishRoot, fileName));
|
|
await verifyCurrentPublication(safePublishRoot, mergedAssets);
|
|
await mkdir(dirname(manifestPath), { recursive: true });
|
|
await writeFile(manifestPath, `${JSON.stringify({ assets: mergedAssets }, null, 2)}\n`, 'utf8');
|
|
};
|
|
|
|
export const prepareAssets = async (options) => {
|
|
const required = ['sourceRoot', 'mappingPath', 'denylistPath', 'recipesPath', 'stagingRoot', 'reportDir'];
|
|
if (!options || required.some((key) => typeof options[key] !== 'string')) fail('ASSET-ARGUMENT-001');
|
|
const recipes = await loadRecipes(options.recipesPath);
|
|
const mapping = await loadPrivateMapping(options.mappingPath, recipes);
|
|
const denylist = await loadDenylist(options.denylistPath);
|
|
const stagingRoot = await resetDirectory(options.stagingRoot, 'ASSET-STAGING-001');
|
|
const reportDir = await resetDirectory(options.reportDir, 'ASSET-REPORT-001');
|
|
const candidates = [];
|
|
for (const recipe of recipes) {
|
|
const source = await resolvePrivateSource(options.sourceRoot, mapping[recipe.id]);
|
|
const prepared = recipe.kind === 'image'
|
|
? await prepareImage({ source, recipe, stagingRoot, denylist })
|
|
: await prepareVideo({ source, recipe, stagingRoot, reportDir, denylist });
|
|
candidates.push(...prepared);
|
|
}
|
|
const finalCandidates = sorted(candidates);
|
|
const approval = approvalProjection(recipes, finalCandidates);
|
|
await writeFile(join(reportDir, 'candidate-approval.json'), `${JSON.stringify(approval, null, 2)}\n`, 'utf8');
|
|
await writeFile(join(reportDir, 'scan-report.json'), `${JSON.stringify({
|
|
schema: 'portfolio-private-scan-report/v1', automatedChecks: 'passed', publication: 'pending-manual-approval',
|
|
assets: finalCandidates.map(({ id, fileName, sha256 }) => ({ id, fileName, sha256 })),
|
|
}, null, 2)}\n`, 'utf8');
|
|
await createContactSheet(finalCandidates, stagingRoot, reportDir);
|
|
let approvalStatus = 'pending';
|
|
if (options.approvalRegisterPath !== undefined) {
|
|
if (typeof options.publishRoot !== 'string' || typeof options.manifestPath !== 'string') fail('ASSET-ARGUMENT-001');
|
|
await approvalsMatch(options.approvalRegisterPath, approval);
|
|
await publish({
|
|
candidates: finalCandidates, recipes, stagingRoot,
|
|
publishRoot: options.publishRoot, manifestPath: options.manifestPath,
|
|
});
|
|
approvalStatus = 'approved';
|
|
}
|
|
return { candidates: finalCandidates, approvalStatus, reportDir };
|
|
};
|
|
|
|
export const verifyPreparedAssets = async ({ publishRoot, manifestPath, denylistPath }) => {
|
|
if (![publishRoot, manifestPath, denylistPath].every((value) => typeof value === 'string')) fail('ASSET-ARGUMENT-001');
|
|
const assets = await loadManifestAssets(manifestPath);
|
|
const safePublishRoot = await verifyCurrentPublication(publishRoot, assets);
|
|
const denylist = await loadDenylist(denylistPath);
|
|
for (const item of assets) {
|
|
const path = join(safePublishRoot, item.fileName);
|
|
if (item.mediaType.startsWith('image/')) await scanImage(path, denylist, { runOcr: true });
|
|
else if (item.mediaType.startsWith('video/')) await scanBinary(path, denylist);
|
|
}
|
|
return true;
|
|
};
|
|
|
|
const parseArgs = (argv) => {
|
|
const values = {};
|
|
let verify = false;
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const key = argv[index];
|
|
if (key === '--verify') { verify = true; continue; }
|
|
if (!key?.startsWith('--') || index + 1 >= argv.length) fail('ASSET-ARGUMENT-001');
|
|
values[key.slice(2)] = argv[index + 1];
|
|
index += 1;
|
|
}
|
|
return { verify, values };
|
|
};
|
|
|
|
const runCli = async () => {
|
|
const { verify, values } = parseArgs(process.argv.slice(2));
|
|
if (verify) {
|
|
await verifyPreparedAssets({ publishRoot: values['publish-root'], manifestPath: values.manifest, denylistPath: values.denylist });
|
|
return;
|
|
}
|
|
await prepareAssets({
|
|
sourceRoot: values['source-root'], mappingPath: values.mapping, denylistPath: values.denylist,
|
|
recipesPath: values.recipes, stagingRoot: values['staging-root'], reportDir: values['report-dir'],
|
|
approvalRegisterPath: values.approvals, publishRoot: values['publish-root'], manifestPath: values.manifest,
|
|
});
|
|
};
|
|
|
|
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
runCli().catch((error) => {
|
|
const message = error instanceof Error && /^[A-Z]+(?:-[A-Z]+)*-\d{3}(?::[a-z0-9-]+)?$/u.test(error.message)
|
|
? error.message
|
|
: 'ASSET-UNEXPECTED-001';
|
|
process.stderr.write(`${message}\n`);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|