mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
* refactor(tooling): group remaining scripts by concrete domain owners * fix(tooling): preserve relocated launcher resource roots
397 lines
14 KiB
JavaScript
397 lines
14 KiB
JavaScript
import { execFileSync } from 'node:child_process'
|
||
import { createHash } from 'node:crypto'
|
||
import { lstat, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'
|
||
import path from 'node:path'
|
||
import process from 'node:process'
|
||
import { resolvePullRequestDiffBase } from '../../git/pr-diff-base/git-pull-request-diff-base.mjs'
|
||
import skillRenames from '../../../../src/shared/bundled-skill-renames.json' with { type: 'json' }
|
||
|
||
// Schema 3 removed revision/tree identity with snapshot registry; freshness compares installed bytes only.
|
||
const CURRENT_MANIFEST_SCHEMA_VERSION = 3
|
||
const SCRIPT_DIR = import.meta.dirname
|
||
const REPO_ROOT = path.resolve(SCRIPT_DIR, '../../..', '..')
|
||
const SKILLS_ROOT = path.join(REPO_ROOT, 'skills')
|
||
const OUTPUT_ROOT = path.join(REPO_ROOT, 'resources', 'skills')
|
||
const CURRENT_MANIFEST_PATH = path.join(OUTPUT_ROOT, 'current-manifest.json')
|
||
// Digest ledger is add-only or older untouched SKILL.md installs become unrecognizable.
|
||
const RELEASED_DIGESTS_PATH = path.join(OUTPUT_ROOT, 'released-digests.json')
|
||
const BUNDLED_SOURCES_PATH = path.join(
|
||
REPO_ROOT,
|
||
'src',
|
||
'main',
|
||
'skills',
|
||
'bundled-skill-install',
|
||
'bundled-skill-sources.ts'
|
||
)
|
||
|
||
function sha256(bytes) {
|
||
return createHash('sha256').update(bytes).digest('hex')
|
||
}
|
||
|
||
function compareCodeUnits(left, right) {
|
||
return left === right ? 0 : left < right ? -1 : 1
|
||
}
|
||
|
||
function normalizeText(bytes) {
|
||
const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
|
||
return Buffer.from(text.replace(/\r\n/g, '\n').replace(/\r/g, '\n'), 'utf8')
|
||
}
|
||
|
||
function normalizedTextOrNull(bytes) {
|
||
if (bytes.includes(0)) {
|
||
return null
|
||
}
|
||
try {
|
||
return normalizeText(bytes)
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
const classifyFile = (bytes) => (normalizedTextOrNull(bytes) === null ? 'binary' : 'text')
|
||
|
||
function assertSafeRelativePath(relativePath) {
|
||
if (
|
||
path.isAbsolute(relativePath) ||
|
||
relativePath === '..' ||
|
||
relativePath.startsWith(`..${path.sep}`)
|
||
) {
|
||
throw new Error(`Unsafe skill package path: ${relativePath}`)
|
||
}
|
||
}
|
||
|
||
function describeFile(manifestPath, bytes, executable) {
|
||
const normalized = normalizedTextOrNull(bytes)
|
||
const exactSha256 = sha256(bytes)
|
||
const textNormalizedSha256 = normalized === null ? null : sha256(normalized)
|
||
return {
|
||
path: manifestPath,
|
||
size: bytes.length,
|
||
executable,
|
||
classification: normalized === null ? 'binary' : 'text',
|
||
exactSha256,
|
||
textNormalizedSha256,
|
||
identitySha256: normalized !== null && !executable ? textNormalizedSha256 : exactSha256
|
||
}
|
||
}
|
||
|
||
// Mirror runtime OS-sidecar exclusions (skill-package-identity.ts) or manifest can describe content no live install matches.
|
||
// Skip OS-authored noise rather than failing builds for browsing a folder.
|
||
const OS_METADATA_FILE_NAMES = new Set(['.ds_store', 'thumbs.db', 'ehthumbs.db', 'desktop.ini'])
|
||
|
||
function isOsMetadataSkillEntryName(name) {
|
||
const folded = name.toLocaleLowerCase('en-US')
|
||
return OS_METADATA_FILE_NAMES.has(folded) || folded.startsWith('._')
|
||
}
|
||
|
||
async function collectPackageFiles(packageRoot) {
|
||
const files = []
|
||
const caseFoldedPaths = new Map()
|
||
|
||
async function visit(directory) {
|
||
const entries = await readdir(directory, { withFileTypes: true })
|
||
// Build Node and packaged Electron may differ in ICU; use locale-independent identity ordering.
|
||
entries.sort((left, right) => compareCodeUnits(left.name, right.name))
|
||
for (const entry of entries) {
|
||
const absolutePath = path.join(directory, entry.name)
|
||
const fileStat = await lstat(absolutePath)
|
||
// Skip sidecar-named plain files only: directories/links contain real content. Check before case-fold collision map.
|
||
if (isOsMetadataSkillEntryName(entry.name) && fileStat.isFile()) {
|
||
continue
|
||
}
|
||
const relativePath = path.relative(packageRoot, absolutePath)
|
||
assertSafeRelativePath(relativePath)
|
||
const manifestPath = relativePath.split(path.sep).join('/')
|
||
const foldedPath = manifestPath.toLocaleLowerCase('en-US')
|
||
const collision = caseFoldedPaths.get(foldedPath)
|
||
if (collision && collision !== manifestPath) {
|
||
throw new Error(`Case-colliding skill paths: ${collision} and ${manifestPath}`)
|
||
}
|
||
caseFoldedPaths.set(foldedPath, manifestPath)
|
||
if (fileStat.isSymbolicLink()) {
|
||
throw new Error(`Symlink is not allowed in a shipped skill: ${manifestPath}`)
|
||
}
|
||
if (fileStat.isDirectory()) {
|
||
await visit(absolutePath)
|
||
continue
|
||
}
|
||
if (!fileStat.isFile()) {
|
||
throw new Error(`Special file is not allowed in a shipped skill: ${manifestPath}`)
|
||
}
|
||
// Windows cannot observe execute bits; shipped executables would make pristine installs appear outdated.
|
||
if ((fileStat.mode & 0o111) !== 0) {
|
||
throw new Error(`Executable file is not allowed in a shipped skill: ${manifestPath}`)
|
||
}
|
||
files.push(describeFile(manifestPath, await readFile(absolutePath), false))
|
||
}
|
||
}
|
||
|
||
await visit(packageRoot)
|
||
return sortManifestFiles(files)
|
||
}
|
||
|
||
// Digest hashes array order; match runtime’s sorted depth-first filesystem walk.
|
||
function compareManifestPaths(left, right) {
|
||
const leftParts = left.split('/')
|
||
const rightParts = right.split('/')
|
||
const shared = Math.min(leftParts.length, rightParts.length)
|
||
for (let index = 0; index < shared; index += 1) {
|
||
const order = compareCodeUnits(leftParts[index], rightParts[index])
|
||
if (order !== 0) {
|
||
return order
|
||
}
|
||
}
|
||
return leftParts.length - rightParts.length
|
||
}
|
||
|
||
function sortManifestFiles(files) {
|
||
return [...files].sort((left, right) => compareManifestPaths(left.path, right.path))
|
||
}
|
||
|
||
function packageDigest(files) {
|
||
return sha256(
|
||
Buffer.from(
|
||
JSON.stringify(
|
||
files.map((file) => ({
|
||
path: file.path,
|
||
executable: file.executable,
|
||
classification: file.classification,
|
||
identitySha256: file.identitySha256
|
||
}))
|
||
),
|
||
'utf8'
|
||
)
|
||
)
|
||
}
|
||
|
||
async function buildCurrentManifest() {
|
||
const skillDirectories = (await readdir(SKILLS_ROOT, { withFileTypes: true }))
|
||
.filter((entry) => entry.isDirectory())
|
||
.map((entry) => entry.name)
|
||
.sort(compareCodeUnits)
|
||
const skills = []
|
||
for (const name of skillDirectories) {
|
||
const files = await collectPackageFiles(path.join(SKILLS_ROOT, name))
|
||
if (!files.some((file) => file.path === 'SKILL.md')) {
|
||
throw new Error(`Skill package ${name} has no top-level SKILL.md`)
|
||
}
|
||
skills.push({ name, sourcePath: `skills/${name}`, packageDigest: packageDigest(files), files })
|
||
}
|
||
return { schemaVersion: CURRENT_MANIFEST_SCHEMA_VERSION, skills }
|
||
}
|
||
|
||
function serialized(value) {
|
||
return `${JSON.stringify(value, null, 2)}\n`
|
||
}
|
||
|
||
function textDigest(bytes) {
|
||
const normalized = normalizedTextOrNull(bytes)
|
||
return normalized === null ? null : sha256(normalized)
|
||
}
|
||
|
||
function git(args, options = {}) {
|
||
return execFileSync('git', args, {
|
||
cwd: REPO_ROOT,
|
||
maxBuffer: 64 * 1024 * 1024,
|
||
stdio: ['pipe', 'pipe', 'pipe'],
|
||
...options
|
||
})
|
||
}
|
||
|
||
function tryGit(args, options) {
|
||
try {
|
||
return git(args, { encoding: 'utf8', ...options })
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
const SKILL_FILE_RAW_LINE = /^:\d+ \d+ [0-9a-f]+ ([0-9a-f]+) [A-Z]\d*\tskills\/([^/]+)\/SKILL\.md$/u
|
||
|
||
/** Every `skills/<name>/SKILL.md` blob reachable from HEAD or a release tag; empty outside a git checkout. */
|
||
function historicalSkillDigests() {
|
||
// One path-limited log plus one batched read: never a per-commit or per-ref fan-out.
|
||
const logFrom = (refs) =>
|
||
tryGit(['log', '--format=', '--raw', '--no-abbrev', '--no-renames', ...refs, '--', 'skills'])
|
||
// Why tags: a release cut off a branch ships bytes HEAD may never contain; shallow CI simply has none.
|
||
const log = logFrom(['HEAD', '--tags=v*']) ?? logFrom(['HEAD'])
|
||
if (log === null) {
|
||
return []
|
||
}
|
||
const namesByBlob = new Map()
|
||
for (const line of log.split('\n')) {
|
||
const match = SKILL_FILE_RAW_LINE.exec(line)
|
||
if (match && !/^0+$/u.test(match[1])) {
|
||
namesByBlob.set(match[1], match[2])
|
||
}
|
||
}
|
||
if (namesByBlob.size === 0) {
|
||
return []
|
||
}
|
||
const batch = git(['cat-file', '--batch'], { input: `${[...namesByBlob.keys()].join('\n')}\n` })
|
||
const digests = []
|
||
for (let offset = 0; offset < batch.length;) {
|
||
const headerEnd = batch.indexOf(0x0a, offset)
|
||
const [blob, , size] = batch.subarray(offset, headerEnd).toString('utf8').split(' ')
|
||
const bytes = batch.subarray(headerEnd + 1, headerEnd + 1 + Number(size))
|
||
offset = headerEnd + 1 + Number(size) + 1
|
||
const digest = textDigest(bytes)
|
||
if (digest !== null) {
|
||
digests.push([namesByBlob.get(blob), [digest]])
|
||
}
|
||
}
|
||
return digests
|
||
}
|
||
|
||
function withDigests(ledger, additions) {
|
||
const skills = { ...ledger.skills }
|
||
for (const [name, digests] of additions) {
|
||
skills[name] = [...new Set([...(skills[name] ?? []), ...digests])].sort(compareCodeUnits)
|
||
}
|
||
const names = Object.keys(skills).sort(compareCodeUnits)
|
||
return { schemaVersion: 1, skills: Object.fromEntries(names.map((name) => [name, skills[name]])) }
|
||
}
|
||
|
||
/** Digests a base ledger listed that this one lost. */
|
||
function findDroppedReleasedDigests(baseLedger, ledger) {
|
||
return Object.entries(baseLedger.skills ?? {}).flatMap(([name, digests]) =>
|
||
digests
|
||
.filter((digest) => !(ledger.skills[name] ?? []).includes(digest))
|
||
.map((digest) => `${name}:${digest}`)
|
||
)
|
||
}
|
||
|
||
async function readReleasedDigests() {
|
||
const text = await readFile(RELEASED_DIGESTS_PATH, 'utf8').catch(() => null)
|
||
return text === null ? { schemaVersion: 1, skills: {} } : JSON.parse(text)
|
||
}
|
||
|
||
async function buildReleasedDigests(manifest) {
|
||
const current = []
|
||
for (const skill of manifest.skills) {
|
||
const bytes = await readFile(path.join(REPO_ROOT, skill.sourcePath, 'SKILL.md'))
|
||
current.push([skill.name, [textDigest(bytes)]])
|
||
}
|
||
return withDigests(await readReleasedDigests(), [...historicalSkillDigests(), ...current])
|
||
}
|
||
|
||
// Embed install bytes: CLI/paired servers have no app resources directory.
|
||
async function buildBundledSources(manifest, ledger) {
|
||
const sources = []
|
||
for (const skill of manifest.skills) {
|
||
// The installer's untouched-copy rule is "the folder holds exactly SKILL.md".
|
||
if (skill.files.length !== 1 || skill.files[0].classification !== 'text') {
|
||
throw new Error(`Bundled skills ship a single text SKILL.md: ${skill.name}`)
|
||
}
|
||
const bytes = await readFile(path.join(REPO_ROOT, skill.sourcePath, 'SKILL.md'))
|
||
sources.push({
|
||
name: skill.name,
|
||
text: normalizeText(bytes).toString('utf8'),
|
||
...(skillRenames[skill.name] ? { aliases: skillRenames[skill.name] } : {}),
|
||
releasedDigests: [
|
||
...new Set(
|
||
[skill.name, ...(skillRenames[skill.name] ?? [])].flatMap(
|
||
(name) => ledger.skills[name] ?? []
|
||
)
|
||
)
|
||
].sort(compareCodeUnits)
|
||
})
|
||
}
|
||
return (
|
||
'// Generated by config/scripts/generate-skill-bundle-manifest.mjs. Do not edit.\n\n' +
|
||
'export type BundledSkillSource = {\n' +
|
||
' readonly name: string\n' +
|
||
' readonly aliases?: readonly string[]\n' +
|
||
' /** LF-normalised `SKILL.md`, the only file a bundled skill ships. */\n' +
|
||
' readonly text: string\n' +
|
||
' /** sha256 of every LF-normalised `SKILL.md` any build shipped, this one included. */\n' +
|
||
' readonly releasedDigests: readonly string[]\n' +
|
||
'}\n\n' +
|
||
'// oxfmt-ignore\n' +
|
||
'export const BUNDLED_SKILL_SOURCES: readonly BundledSkillSource[] = [\n' +
|
||
`${sources.map((source) => ` ${JSON.stringify(source)}`).join(',\n')}\n]\n`
|
||
)
|
||
}
|
||
|
||
// Why: on a PR merge ref HEAD^1 is the base; locally, the merge base with main.
|
||
function ledgerBaseCommit() {
|
||
try {
|
||
return (
|
||
resolvePullRequestDiffBase(REPO_ROOT, null) ??
|
||
(tryGit(['merge-base', 'origin/main', 'HEAD'])?.trim() || null)
|
||
)
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
function verifyLedgerKeepsBase(ledger) {
|
||
const base = ledgerBaseCommit()
|
||
const relativePath = path.relative(REPO_ROOT, RELEASED_DIGESTS_PATH).split(path.sep).join('/')
|
||
// Why fail closed on a PR: an unreadable base would otherwise let a shrink through silently.
|
||
if (
|
||
process.env.GITHUB_EVENT_NAME === 'pull_request' &&
|
||
(!base || tryGit(['cat-file', '-e', base]) === null)
|
||
) {
|
||
throw new Error(`Cannot read the pull request base to check ${relativePath} only grows.`)
|
||
}
|
||
const baseText = base ? tryGit(['show', `${base}:${relativePath}`]) : null
|
||
if (baseText === null) {
|
||
return
|
||
}
|
||
const dropped = findDroppedReleasedDigests(JSON.parse(baseText), ledger)
|
||
if (dropped.length > 0) {
|
||
throw new Error(
|
||
`${relativePath} dropped digests its base shipped (it is add-only):\n${dropped.join('\n')}`
|
||
)
|
||
}
|
||
}
|
||
|
||
async function verifyGeneratedFile(filePath, expected) {
|
||
const committedText = await readFile(filePath, 'utf8').catch(() => null)
|
||
if (committedText !== expected) {
|
||
throw new Error(
|
||
`Generated skill bundle artifact is stale:\n${path.relative(REPO_ROOT, filePath)}\n` +
|
||
'Run pnpm generate:skill-bundle-manifest.'
|
||
)
|
||
}
|
||
}
|
||
|
||
async function main() {
|
||
const manifest = await buildCurrentManifest()
|
||
const ledger = await buildReleasedDigests(manifest)
|
||
const sources = await buildBundledSources(manifest, ledger)
|
||
if (process.argv.slice(2).includes('--write')) {
|
||
await mkdir(OUTPUT_ROOT, { recursive: true })
|
||
await mkdir(path.dirname(BUNDLED_SOURCES_PATH), { recursive: true })
|
||
await writeFile(CURRENT_MANIFEST_PATH, serialized(manifest))
|
||
await writeFile(RELEASED_DIGESTS_PATH, serialized(ledger))
|
||
await writeFile(BUNDLED_SOURCES_PATH, sources)
|
||
return
|
||
}
|
||
await verifyGeneratedFile(CURRENT_MANIFEST_PATH, serialized(manifest))
|
||
// Equal to its own union with history and the current bytes means none is missing.
|
||
await verifyGeneratedFile(RELEASED_DIGESTS_PATH, serialized(ledger))
|
||
verifyLedgerKeepsBase(ledger)
|
||
await verifyGeneratedFile(BUNDLED_SOURCES_PATH, sources)
|
||
}
|
||
|
||
if (process.argv[1] && path.resolve(process.argv[1]) === import.meta.filename) {
|
||
main().catch((error) => {
|
||
console.error(error instanceof Error ? error.message : error)
|
||
process.exitCode = 1
|
||
})
|
||
}
|
||
|
||
export {
|
||
buildCurrentManifest,
|
||
classifyFile,
|
||
collectPackageFiles,
|
||
describeFile,
|
||
findDroppedReleasedDigests,
|
||
normalizeText,
|
||
packageDigest,
|
||
sortManifestFiles,
|
||
withDigests
|
||
}
|