Files
orca/config/scripts/skills/bundle-generation/generate-skill-bundle-manifest.mjs
Neil 6bb2e13c35 refactor(tooling): organize remaining scripts by domain owners (#38407)
* refactor(tooling): group remaining scripts by concrete domain owners

* fix(tooling): preserve relocated launcher resource roots
2026-11-05 00:29:16 -08:00

397 lines
14 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { lstat, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'
import path from 'node:path'
import process from 'node:process'
import { resolvePullRequestDiffBase } from '../../git/pr-diff-base/git-pull-request-diff-base.mjs'
import skillRenames from '../../../../src/shared/bundled-skill-renames.json' with { type: 'json' }
// Schema 3 removed revision/tree identity with snapshot registry; freshness compares installed bytes only.
const CURRENT_MANIFEST_SCHEMA_VERSION = 3
const SCRIPT_DIR = import.meta.dirname
const REPO_ROOT = path.resolve(SCRIPT_DIR, '../../..', '..')
const SKILLS_ROOT = path.join(REPO_ROOT, 'skills')
const OUTPUT_ROOT = path.join(REPO_ROOT, 'resources', 'skills')
const CURRENT_MANIFEST_PATH = path.join(OUTPUT_ROOT, 'current-manifest.json')
// Digest ledger is add-only or older untouched SKILL.md installs become unrecognizable.
const RELEASED_DIGESTS_PATH = path.join(OUTPUT_ROOT, 'released-digests.json')
const BUNDLED_SOURCES_PATH = path.join(
REPO_ROOT,
'src',
'main',
'skills',
'bundled-skill-install',
'bundled-skill-sources.ts'
)
function sha256(bytes) {
return createHash('sha256').update(bytes).digest('hex')
}
function compareCodeUnits(left, right) {
return left === right ? 0 : left < right ? -1 : 1
}
function normalizeText(bytes) {
const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
return Buffer.from(text.replace(/\r\n/g, '\n').replace(/\r/g, '\n'), 'utf8')
}
function normalizedTextOrNull(bytes) {
if (bytes.includes(0)) {
return null
}
try {
return normalizeText(bytes)
} catch {
return null
}
}
const classifyFile = (bytes) => (normalizedTextOrNull(bytes) === null ? 'binary' : 'text')
function assertSafeRelativePath(relativePath) {
if (
path.isAbsolute(relativePath) ||
relativePath === '..' ||
relativePath.startsWith(`..${path.sep}`)
) {
throw new Error(`Unsafe skill package path: ${relativePath}`)
}
}
function describeFile(manifestPath, bytes, executable) {
const normalized = normalizedTextOrNull(bytes)
const exactSha256 = sha256(bytes)
const textNormalizedSha256 = normalized === null ? null : sha256(normalized)
return {
path: manifestPath,
size: bytes.length,
executable,
classification: normalized === null ? 'binary' : 'text',
exactSha256,
textNormalizedSha256,
identitySha256: normalized !== null && !executable ? textNormalizedSha256 : exactSha256
}
}
// Mirror runtime OS-sidecar exclusions (skill-package-identity.ts) or manifest can describe content no live install matches.
// Skip OS-authored noise rather than failing builds for browsing a folder.
const OS_METADATA_FILE_NAMES = new Set(['.ds_store', 'thumbs.db', 'ehthumbs.db', 'desktop.ini'])
function isOsMetadataSkillEntryName(name) {
const folded = name.toLocaleLowerCase('en-US')
return OS_METADATA_FILE_NAMES.has(folded) || folded.startsWith('._')
}
async function collectPackageFiles(packageRoot) {
const files = []
const caseFoldedPaths = new Map()
async function visit(directory) {
const entries = await readdir(directory, { withFileTypes: true })
// Build Node and packaged Electron may differ in ICU; use locale-independent identity ordering.
entries.sort((left, right) => compareCodeUnits(left.name, right.name))
for (const entry of entries) {
const absolutePath = path.join(directory, entry.name)
const fileStat = await lstat(absolutePath)
// Skip sidecar-named plain files only: directories/links contain real content. Check before case-fold collision map.
if (isOsMetadataSkillEntryName(entry.name) && fileStat.isFile()) {
continue
}
const relativePath = path.relative(packageRoot, absolutePath)
assertSafeRelativePath(relativePath)
const manifestPath = relativePath.split(path.sep).join('/')
const foldedPath = manifestPath.toLocaleLowerCase('en-US')
const collision = caseFoldedPaths.get(foldedPath)
if (collision && collision !== manifestPath) {
throw new Error(`Case-colliding skill paths: ${collision} and ${manifestPath}`)
}
caseFoldedPaths.set(foldedPath, manifestPath)
if (fileStat.isSymbolicLink()) {
throw new Error(`Symlink is not allowed in a shipped skill: ${manifestPath}`)
}
if (fileStat.isDirectory()) {
await visit(absolutePath)
continue
}
if (!fileStat.isFile()) {
throw new Error(`Special file is not allowed in a shipped skill: ${manifestPath}`)
}
// Windows cannot observe execute bits; shipped executables would make pristine installs appear outdated.
if ((fileStat.mode & 0o111) !== 0) {
throw new Error(`Executable file is not allowed in a shipped skill: ${manifestPath}`)
}
files.push(describeFile(manifestPath, await readFile(absolutePath), false))
}
}
await visit(packageRoot)
return sortManifestFiles(files)
}
// Digest hashes array order; match runtime’s sorted depth-first filesystem walk.
function compareManifestPaths(left, right) {
const leftParts = left.split('/')
const rightParts = right.split('/')
const shared = Math.min(leftParts.length, rightParts.length)
for (let index = 0; index < shared; index += 1) {
const order = compareCodeUnits(leftParts[index], rightParts[index])
if (order !== 0) {
return order
}
}
return leftParts.length - rightParts.length
}
function sortManifestFiles(files) {
return [...files].sort((left, right) => compareManifestPaths(left.path, right.path))
}
function packageDigest(files) {
return sha256(
Buffer.from(
JSON.stringify(
files.map((file) => ({
path: file.path,
executable: file.executable,
classification: file.classification,
identitySha256: file.identitySha256
}))
),
'utf8'
)
)
}
async function buildCurrentManifest() {
const skillDirectories = (await readdir(SKILLS_ROOT, { withFileTypes: true }))
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort(compareCodeUnits)
const skills = []
for (const name of skillDirectories) {
const files = await collectPackageFiles(path.join(SKILLS_ROOT, name))
if (!files.some((file) => file.path === 'SKILL.md')) {
throw new Error(`Skill package ${name} has no top-level SKILL.md`)
}
skills.push({ name, sourcePath: `skills/${name}`, packageDigest: packageDigest(files), files })
}
return { schemaVersion: CURRENT_MANIFEST_SCHEMA_VERSION, skills }
}
function serialized(value) {
return `${JSON.stringify(value, null, 2)}\n`
}
function textDigest(bytes) {
const normalized = normalizedTextOrNull(bytes)
return normalized === null ? null : sha256(normalized)
}
function git(args, options = {}) {
return execFileSync('git', args, {
cwd: REPO_ROOT,
maxBuffer: 64 * 1024 * 1024,
stdio: ['pipe', 'pipe', 'pipe'],
...options
})
}
function tryGit(args, options) {
try {
return git(args, { encoding: 'utf8', ...options })
} catch {
return null
}
}
const SKILL_FILE_RAW_LINE = /^:\d+ \d+ [0-9a-f]+ ([0-9a-f]+) [A-Z]\d*\tskills\/([^/]+)\/SKILL\.md$/u
/** Every `skills/<name>/SKILL.md` blob reachable from HEAD or a release tag; empty outside a git checkout. */
function historicalSkillDigests() {
// One path-limited log plus one batched read: never a per-commit or per-ref fan-out.
const logFrom = (refs) =>
tryGit(['log', '--format=', '--raw', '--no-abbrev', '--no-renames', ...refs, '--', 'skills'])
// Why tags: a release cut off a branch ships bytes HEAD may never contain; shallow CI simply has none.
const log = logFrom(['HEAD', '--tags=v*']) ?? logFrom(['HEAD'])
if (log === null) {
return []
}
const namesByBlob = new Map()
for (const line of log.split('\n')) {
const match = SKILL_FILE_RAW_LINE.exec(line)
if (match && !/^0+$/u.test(match[1])) {
namesByBlob.set(match[1], match[2])
}
}
if (namesByBlob.size === 0) {
return []
}
const batch = git(['cat-file', '--batch'], { input: `${[...namesByBlob.keys()].join('\n')}\n` })
const digests = []
for (let offset = 0; offset < batch.length;) {
const headerEnd = batch.indexOf(0x0a, offset)
const [blob, , size] = batch.subarray(offset, headerEnd).toString('utf8').split(' ')
const bytes = batch.subarray(headerEnd + 1, headerEnd + 1 + Number(size))
offset = headerEnd + 1 + Number(size) + 1
const digest = textDigest(bytes)
if (digest !== null) {
digests.push([namesByBlob.get(blob), [digest]])
}
}
return digests
}
function withDigests(ledger, additions) {
const skills = { ...ledger.skills }
for (const [name, digests] of additions) {
skills[name] = [...new Set([...(skills[name] ?? []), ...digests])].sort(compareCodeUnits)
}
const names = Object.keys(skills).sort(compareCodeUnits)
return { schemaVersion: 1, skills: Object.fromEntries(names.map((name) => [name, skills[name]])) }
}
/** Digests a base ledger listed that this one lost. */
function findDroppedReleasedDigests(baseLedger, ledger) {
return Object.entries(baseLedger.skills ?? {}).flatMap(([name, digests]) =>
digests
.filter((digest) => !(ledger.skills[name] ?? []).includes(digest))
.map((digest) => `${name}:${digest}`)
)
}
async function readReleasedDigests() {
const text = await readFile(RELEASED_DIGESTS_PATH, 'utf8').catch(() => null)
return text === null ? { schemaVersion: 1, skills: {} } : JSON.parse(text)
}
async function buildReleasedDigests(manifest) {
const current = []
for (const skill of manifest.skills) {
const bytes = await readFile(path.join(REPO_ROOT, skill.sourcePath, 'SKILL.md'))
current.push([skill.name, [textDigest(bytes)]])
}
return withDigests(await readReleasedDigests(), [...historicalSkillDigests(), ...current])
}
// Embed install bytes: CLI/paired servers have no app resources directory.
async function buildBundledSources(manifest, ledger) {
const sources = []
for (const skill of manifest.skills) {
// The installer's untouched-copy rule is "the folder holds exactly SKILL.md".
if (skill.files.length !== 1 || skill.files[0].classification !== 'text') {
throw new Error(`Bundled skills ship a single text SKILL.md: ${skill.name}`)
}
const bytes = await readFile(path.join(REPO_ROOT, skill.sourcePath, 'SKILL.md'))
sources.push({
name: skill.name,
text: normalizeText(bytes).toString('utf8'),
...(skillRenames[skill.name] ? { aliases: skillRenames[skill.name] } : {}),
releasedDigests: [
...new Set(
[skill.name, ...(skillRenames[skill.name] ?? [])].flatMap(
(name) => ledger.skills[name] ?? []
)
)
].sort(compareCodeUnits)
})
}
return (
'// Generated by config/scripts/generate-skill-bundle-manifest.mjs. Do not edit.\n\n' +
'export type BundledSkillSource = {\n' +
' readonly name: string\n' +
' readonly aliases?: readonly string[]\n' +
' /** LF-normalised `SKILL.md`, the only file a bundled skill ships. */\n' +
' readonly text: string\n' +
' /** sha256 of every LF-normalised `SKILL.md` any build shipped, this one included. */\n' +
' readonly releasedDigests: readonly string[]\n' +
'}\n\n' +
'// oxfmt-ignore\n' +
'export const BUNDLED_SKILL_SOURCES: readonly BundledSkillSource[] = [\n' +
`${sources.map((source) => ` ${JSON.stringify(source)}`).join(',\n')}\n]\n`
)
}
// Why: on a PR merge ref HEAD^1 is the base; locally, the merge base with main.
function ledgerBaseCommit() {
try {
return (
resolvePullRequestDiffBase(REPO_ROOT, null) ??
(tryGit(['merge-base', 'origin/main', 'HEAD'])?.trim() || null)
)
} catch {
return null
}
}
function verifyLedgerKeepsBase(ledger) {
const base = ledgerBaseCommit()
const relativePath = path.relative(REPO_ROOT, RELEASED_DIGESTS_PATH).split(path.sep).join('/')
// Why fail closed on a PR: an unreadable base would otherwise let a shrink through silently.
if (
process.env.GITHUB_EVENT_NAME === 'pull_request' &&
(!base || tryGit(['cat-file', '-e', base]) === null)
) {
throw new Error(`Cannot read the pull request base to check ${relativePath} only grows.`)
}
const baseText = base ? tryGit(['show', `${base}:${relativePath}`]) : null
if (baseText === null) {
return
}
const dropped = findDroppedReleasedDigests(JSON.parse(baseText), ledger)
if (dropped.length > 0) {
throw new Error(
`${relativePath} dropped digests its base shipped (it is add-only):\n${dropped.join('\n')}`
)
}
}
async function verifyGeneratedFile(filePath, expected) {
const committedText = await readFile(filePath, 'utf8').catch(() => null)
if (committedText !== expected) {
throw new Error(
`Generated skill bundle artifact is stale:\n${path.relative(REPO_ROOT, filePath)}\n` +
'Run pnpm generate:skill-bundle-manifest.'
)
}
}
async function main() {
const manifest = await buildCurrentManifest()
const ledger = await buildReleasedDigests(manifest)
const sources = await buildBundledSources(manifest, ledger)
if (process.argv.slice(2).includes('--write')) {
await mkdir(OUTPUT_ROOT, { recursive: true })
await mkdir(path.dirname(BUNDLED_SOURCES_PATH), { recursive: true })
await writeFile(CURRENT_MANIFEST_PATH, serialized(manifest))
await writeFile(RELEASED_DIGESTS_PATH, serialized(ledger))
await writeFile(BUNDLED_SOURCES_PATH, sources)
return
}
await verifyGeneratedFile(CURRENT_MANIFEST_PATH, serialized(manifest))
// Equal to its own union with history and the current bytes means none is missing.
await verifyGeneratedFile(RELEASED_DIGESTS_PATH, serialized(ledger))
verifyLedgerKeepsBase(ledger)
await verifyGeneratedFile(BUNDLED_SOURCES_PATH, sources)
}
if (process.argv[1] && path.resolve(process.argv[1]) === import.meta.filename) {
main().catch((error) => {
console.error(error instanceof Error ? error.message : error)
process.exitCode = 1
})
}
export {
buildCurrentManifest,
classifyFile,
collectPackageFiles,
describeFile,
findDroppedReleasedDigests,
normalizeText,
packageDigest,
sortManifestFiles,
withDigests
}