mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-07 00:01:49 +00:00
feat(cli): deduplicate identical script lockfiles (dedupeLockfiles) (#10769)
* feat(cli): deduplicate identical script lockfiles into one per language * test: pin shared lockfile path classification * fix(cli): never delete a lockfile the dedup plan also writes * fix(cli): plan lockfile dedup from the whole tree, not the sync scope * fix(cli): keep dedup out of dry runs and stop hiding scripts from its scan * test: pin which files the shared-lock scan counts as readers * fix(cli): validate shared-lock refs and let the majority keep its file * fix(cli): snapshot shared-lock ownership before regeneration moves it * fix(cli): address dedup review nits (dry-run push, json shape, scan scope) * refactor(cli): put shared lockfiles in a top-level locks/ directory * fix(cli): claim only the shared lock names windmill writes, and only when on * fix(cli): read the lock field itself, and count only scripts sync reads * fix(cli): parse metadata by its real format and lint from the sync root * fix(cli): never re-hash a script whose generation failed * fix(cli): share sync's walk exclusions and fail closed on unreadable dirs * fix(cli): keep a lockfile the metadata on disk still references * fix(cli): decide a lock is unread from the metadata field sync reads * refactor(cli): name shared lockfiles after the dependency file they resolve * fix(cli): carry shared lockfiles a narrowed sync cannot speak for * fix(cli): move a shared lockfile when its dependency file moved, not on a head count * fix(cli): let the many correct a shared lockfile a lone variant planted * fix(cli): read why a lock differs from the stamp the worker writes into it * fix(cli): let an agreeing majority speak whatever the stamps say * docs(cli): count the disjuncts the comment introduces * fix(cli): keep a private lock for any script the worker locks differently * fix(cli): match annotations by the worker's own names, not by shape * fix(cli): recognize the py: interpreter pin the macro does not cover * fix(cli): let the map speak for dependency-file deletions * perf(cli): group lock entries without rebuilding the group per insert * fix(cli): defer shared-lock deletions until the metadata has settled * fix(cli): decide shared-lock readers by the lock field, failing closed * fix(cli): read folded lock refs and keep locks read by unparseable metadata * refactor(cli): one shared-lock reader scan, shared by the pull and push paths * fix(cli): keep nested dependency set names out of shared lockfiles * fix(cli): drop a shared-lock scan gate that no real repo took
This commit is contained in:
Ruben Fiszel
@@ -44,6 +44,222 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
|
||||
{ language: "powershell", filename: "modules.json" },
|
||||
] as const;
|
||||
|
||||
export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined {
|
||||
const relativePath = path.replace("dependencies/", "");
|
||||
for (const { filename, language } of workspaceDependenciesLanguages) {
|
||||
if (relativePath.endsWith(filename)) {
|
||||
return {
|
||||
name: relativePath === filename ? undefined : relativePath.replace("." + filename, ""),
|
||||
language
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Annotation parser — mirrors backend's WorkspaceDependenciesAnnotatedRefs::parse
|
||||
// (windmill-common/src/workspace_dependencies.rs), so the CLI can tell which
|
||||
// workspace dependency file a script resolves against without asking a worker.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type AnnotationMode = "manual" | "extra";
|
||||
|
||||
export interface WorkspaceDepsAnnotation {
|
||||
mode: AnnotationMode;
|
||||
external: string[];
|
||||
inline: string | null;
|
||||
}
|
||||
|
||||
const LANG_ANNOTATION_CONFIG: Partial<
|
||||
Record<ScriptLanguage, { comment: string; keyword: string; validityRe?: RegExp }>
|
||||
> = {
|
||||
python3: { comment: "#", keyword: "requirements", validityRe: /^#\s?(\S+)\s*$/ },
|
||||
bun: { comment: "//", keyword: "package_json" },
|
||||
nativets: { comment: "//", keyword: "package_json" },
|
||||
go: { comment: "//", keyword: "go_mod" },
|
||||
php: { comment: "//", keyword: "composer_json" },
|
||||
powershell: { comment: "#", keyword: "modules_json" },
|
||||
};
|
||||
|
||||
export function extractWorkspaceDepsAnnotation(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
): WorkspaceDepsAnnotation | null {
|
||||
const config = LANG_ANNOTATION_CONFIG[language];
|
||||
if (!config) return null;
|
||||
|
||||
const { comment, keyword, validityRe } = config;
|
||||
const extraMarkerUnderscore = `extra_${keyword}:`;
|
||||
const extraMarkerHyphen = `extra-${keyword}:`;
|
||||
const manualMarker = `${keyword}:`;
|
||||
|
||||
const stripComment = (l: string): string | null => {
|
||||
if (!l.startsWith(comment)) return null;
|
||||
return l.substring(comment.length).trimStart();
|
||||
};
|
||||
const isExtra = (l: string): boolean => {
|
||||
const s = stripComment(l);
|
||||
return s !== null && (s.startsWith(extraMarkerUnderscore) || s.startsWith(extraMarkerHyphen));
|
||||
};
|
||||
const isManual = (l: string): boolean => {
|
||||
const s = stripComment(l);
|
||||
return s !== null && s.startsWith(manualMarker);
|
||||
};
|
||||
|
||||
const lines = scriptContent.split("\n");
|
||||
|
||||
// Find first annotation line (mirrors Rust find_position)
|
||||
let pos = -1;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (isExtra(lines[i]) || isManual(lines[i])) {
|
||||
pos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (pos === -1) return null;
|
||||
|
||||
const annotationLine = lines[pos];
|
||||
const mode: AnnotationMode = isExtra(annotationLine) ? "extra" : "manual";
|
||||
|
||||
// Parse external references from the annotation line
|
||||
const marker = mode === "extra"
|
||||
? (annotationLine.includes(extraMarkerUnderscore) ? extraMarkerUnderscore : extraMarkerHyphen)
|
||||
: manualMarker;
|
||||
const unparsed = annotationLine.replaceAll(marker, "").replaceAll(comment, "");
|
||||
const external = unparsed
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
// Parse inline deps from subsequent lines
|
||||
const inlineParts: string[] = [];
|
||||
for (let i = pos + 1; i < lines.length; i++) {
|
||||
const l = lines[i];
|
||||
if (validityRe) {
|
||||
const match = validityRe.exec(l);
|
||||
if (match && match[1]) {
|
||||
inlineParts.push(match[1]);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (!l.startsWith(comment)) {
|
||||
break;
|
||||
}
|
||||
inlineParts.push(l.substring(comment.length));
|
||||
}
|
||||
}
|
||||
|
||||
const inlineStr = inlineParts.join("\n");
|
||||
const inline = inlineStr.trim().length > 0 ? inlineStr : null;
|
||||
|
||||
return { mode, external, inline };
|
||||
}
|
||||
|
||||
/** The comment marker each language's annotations are written behind. */
|
||||
export const LANG_COMMENT_LIT: Partial<Record<ScriptLanguage, string>> = {
|
||||
python3: "#",
|
||||
ansible: "#",
|
||||
powershell: "#",
|
||||
bun: "//",
|
||||
nativets: "//",
|
||||
deno: "//",
|
||||
go: "//",
|
||||
php: "//",
|
||||
rust: "//!",
|
||||
};
|
||||
|
||||
/**
|
||||
* The annotations each language recognises, by the exact names the worker
|
||||
* matches (`#[annotations(..)]` structs in windmill-common/src/worker.rs).
|
||||
* Several change what it locks — a pinned interpreter, `npm`, `nobundling` —
|
||||
* and the rest are cheap to treat the same way, since the only cost is that
|
||||
* such a script keeps a lockfile of its own.
|
||||
* for related places search: ADD_NEW_LANG
|
||||
*/
|
||||
const LANG_ANNOTATIONS: Partial<Record<ScriptLanguage, string[]>> = {
|
||||
python3: [
|
||||
"no_cache",
|
||||
"no_postinstall",
|
||||
"py_select_latest",
|
||||
"skip_result_postprocessing",
|
||||
"py310",
|
||||
"py311",
|
||||
"py312",
|
||||
"py313",
|
||||
"sandbox",
|
||||
],
|
||||
bun: ["npm", "nodejs", "native", "nobundling", "sandbox"],
|
||||
nativets: ["npm", "nodejs", "native", "nobundling", "sandbox"],
|
||||
deno: ["npm", "nodejs", "native", "nobundling", "sandbox"],
|
||||
go: ["go1_22_compat"],
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether a script's leading comment block carries an annotation the worker
|
||||
* acts on, which means its lock may not be its dependency file's.
|
||||
*
|
||||
* Matched the way the worker matches: the key is the line, or what precedes the
|
||||
* first `=`, and it has to BE one of the names above. Unknown keys are ignored
|
||||
* there and so here — which is what keeps `# TODO:` or `# type: ignore` from
|
||||
* quietly dropping an ordinary documented script out of deduplication.
|
||||
*
|
||||
* `# py: <specifier>` is the exception the macro does not cover: the python
|
||||
* import parser reads it directly (`windmill-parser-py-imports`, alongside the
|
||||
* `py310`..`py313` flags) to pick the interpreter, which changes what resolves.
|
||||
*/
|
||||
export function hasLockAffectingAnnotation(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
): boolean {
|
||||
const comment = LANG_COMMENT_LIT[language];
|
||||
const names = LANG_ANNOTATIONS[language];
|
||||
if (!comment || !names) return false;
|
||||
for (const line of scriptContent.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === "") continue;
|
||||
if (!trimmed.startsWith(comment)) break; // past the header block
|
||||
// Matched on the raw line: the parser tests `# py:`/`#py:` before trimming.
|
||||
if (language === "python3" && /^#\s?py:/.test(line)) return true;
|
||||
const body = trimmed.slice(comment.length).trim();
|
||||
const key = body.split("=")[0].trim();
|
||||
if (names.includes(key)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Where the lockfiles shared by several scripts live when `dedupeLockfiles`
|
||||
* is on — see `utils/lock_dedup.ts`. A top-level directory of its own: what a
|
||||
* group shares is a resolved lock, which needs no workspace dependency file
|
||||
* behind it, and inline-script locks would belong here too. */
|
||||
export const SHARED_LOCK_DIR = "locks";
|
||||
|
||||
/** The lockfile shared by the scripts that resolve against a workspace
|
||||
* dependency file: its own name, plus `.lock`. Appending rather than replacing
|
||||
* the extension keeps the correspondence exact and reversible —
|
||||
* `dependencies/team_a.requirements.in` <-> `locks/team_a.requirements.in.lock`. */
|
||||
export function sharedLockPathFor(depFilePath: string): string {
|
||||
const name = depFilePath.replaceAll("\\", "/").split("/").pop()!;
|
||||
return `${SHARED_LOCK_DIR}/${name}.lock`;
|
||||
}
|
||||
|
||||
/** The workspace dependency file a shared lockfile belongs to, if it is one. */
|
||||
export function depFileOfSharedLock(p: string): string | undefined {
|
||||
const normalized = p.replaceAll("\\", "/");
|
||||
if (!normalized.startsWith(SHARED_LOCK_DIR + "/")) return undefined;
|
||||
const name = normalized.slice(SHARED_LOCK_DIR.length + 1);
|
||||
if (name.includes("/") || !name.endsWith(".lock")) return undefined;
|
||||
const depFile = "dependencies/" + name.slice(0, -".lock".length);
|
||||
const info = workspaceDependenciesPathToLanguageAndFilename(depFile);
|
||||
// `locks/vendor.lock` names no dependency file, so it is not Windmill's: a
|
||||
// repo that already keeps lockfiles here keeps them.
|
||||
return info && languageNeedsLock(info.language) ? depFile : undefined;
|
||||
}
|
||||
|
||||
export function isSharedLockPath(p: string): boolean {
|
||||
return depFileOfSharedLock(p) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if a script in the given language requires a lock file.
|
||||
* Matches the condition in updateScriptLock (metadata.ts).
|
||||
|
||||
Reference in New Issue
Block a user