mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-06 00:02:13 +00:00
feat(cli): deduplicate identical script lockfiles (dedupeLockfiles) (#10769)
* feat(cli): deduplicate identical script lockfiles into one per language * test: pin shared lockfile path classification * fix(cli): never delete a lockfile the dedup plan also writes * fix(cli): plan lockfile dedup from the whole tree, not the sync scope * fix(cli): keep dedup out of dry runs and stop hiding scripts from its scan * test: pin which files the shared-lock scan counts as readers * fix(cli): validate shared-lock refs and let the majority keep its file * fix(cli): snapshot shared-lock ownership before regeneration moves it * fix(cli): address dedup review nits (dry-run push, json shape, scan scope) * refactor(cli): put shared lockfiles in a top-level locks/ directory * fix(cli): claim only the shared lock names windmill writes, and only when on * fix(cli): read the lock field itself, and count only scripts sync reads * fix(cli): parse metadata by its real format and lint from the sync root * fix(cli): never re-hash a script whose generation failed * fix(cli): share sync's walk exclusions and fail closed on unreadable dirs * fix(cli): keep a lockfile the metadata on disk still references * fix(cli): decide a lock is unread from the metadata field sync reads * refactor(cli): name shared lockfiles after the dependency file they resolve * fix(cli): carry shared lockfiles a narrowed sync cannot speak for * fix(cli): move a shared lockfile when its dependency file moved, not on a head count * fix(cli): let the many correct a shared lockfile a lone variant planted * fix(cli): read why a lock differs from the stamp the worker writes into it * fix(cli): let an agreeing majority speak whatever the stamps say * docs(cli): count the disjuncts the comment introduces * fix(cli): keep a private lock for any script the worker locks differently * fix(cli): match annotations by the worker's own names, not by shape * fix(cli): recognize the py: interpreter pin the macro does not cover * fix(cli): let the map speak for dependency-file deletions * perf(cli): group lock entries without rebuilding the group per insert * fix(cli): defer shared-lock deletions until the metadata has settled * fix(cli): decide shared-lock readers by the lock field, failing closed * fix(cli): read folded lock refs and keep locks read by unparseable metadata * refactor(cli): one shared-lock reader scan, shared by the pull and push paths * fix(cli): keep nested dependency set names out of shared lockfiles * fix(cli): drop a shared-lock scan gate that no real repo took
This commit is contained in:
Ruben Fiszel
@@ -0,0 +1,556 @@
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { yamlOptions } from "../commands/sync/sync.ts";
|
||||
import { yamlParseContent } from "./yaml.ts";
|
||||
import {
|
||||
depFileOfSharedLock,
|
||||
extractWorkspaceDepsAnnotation,
|
||||
hasLockAffectingAnnotation,
|
||||
inferContentTypeFromFilePath,
|
||||
isSharedLockPath,
|
||||
languageNeedsLock,
|
||||
sharedLockPathFor,
|
||||
workspaceDependenciesPathToLanguageAndFilename,
|
||||
type ScriptLanguage,
|
||||
} from "./script_common.ts";
|
||||
|
||||
/**
|
||||
* Lockfile deduplication (`dedupeLockfiles` in wmill.yaml).
|
||||
*
|
||||
* A workspace whose dependencies come from `dependencies/<file>` resolves to the
|
||||
* very same lock for every script of that language, so the repo ends up holding
|
||||
* thousands of byte-identical `.script.lock` files: one dependency bump rewrites
|
||||
* all of them, and every open branch conflicts on all of them.
|
||||
*
|
||||
* With dedup on, the scripts that resolve against a workspace dependency file
|
||||
* reference ONE lockfile named after it — `dependencies/requirements.in` ->
|
||||
* `locks/requirements.in.lock` — through the `!inline` indirection their
|
||||
* metadata already uses. A dependency bump is a one-file diff.
|
||||
*
|
||||
* Identity comes from the dependency file, never from the content or from which
|
||||
* scripts happen to be in view. That is what makes the pass stateless: a sync
|
||||
* narrowed to a single script (the git-sync deploy callback) computes the same
|
||||
* NAME as a full one.
|
||||
*
|
||||
* The CONTENT follows, because a group only ever holds scripts whose lock IS
|
||||
* that file's lock: one carrying an annotation the worker acts on — a pinned
|
||||
* interpreter, `npm`, `nobundling` — never joins, so there is no variant inside
|
||||
* a group to tell apart from a bump. What is left is a script whose committed
|
||||
* lock is simply behind, and the many outvote it.
|
||||
*
|
||||
* Two cases are not shared at all and keep a `.script.lock` of their own: a
|
||||
* script whose annotation is `extra_` or carries inline dependencies (its lock
|
||||
* folds in its own imports), and one naming several dependency files at once
|
||||
* (its lock is no single file's).
|
||||
*/
|
||||
|
||||
const INLINE_PREFIX = "!inline ";
|
||||
|
||||
/** Sync maps are keyed with the platform separator, while an `!inline`
|
||||
* reference is always forward-slash. */
|
||||
const toMapKey = (refPath: string) => refPath.replaceAll("/", path.sep);
|
||||
const toRefPath = (mapKey: string) => mapKey.replaceAll("\\", "/");
|
||||
|
||||
/** What the sync layer needs to know to dedup: whether to, and how to read a
|
||||
* `.ts` script's language. Both ride on the same `wmill.yaml` options object. */
|
||||
export type LockDedupOptions = {
|
||||
dedupeLockfiles?: boolean | undefined;
|
||||
defaultTs?: "bun" | "deno" | undefined;
|
||||
};
|
||||
|
||||
/** The files a dedup pass writes and removes, as paths relative to the sync
|
||||
* root — applicable to an in-memory sync map or to the working tree. */
|
||||
export type SharedLockPlan = {
|
||||
writes: Record<string, string>;
|
||||
deletes: string[];
|
||||
};
|
||||
|
||||
export function isEmptySharedLockPlan(plan: SharedLockPlan): boolean {
|
||||
return Object.keys(plan.writes).length === 0 && plan.deletes.length === 0;
|
||||
}
|
||||
|
||||
type ScriptEntry = {
|
||||
metaKey: string;
|
||||
isJson: boolean;
|
||||
compactJson: boolean;
|
||||
parsed: Record<string, any>;
|
||||
/** The lockfile the metadata references today, as a map key. */
|
||||
lockKey: string;
|
||||
/** The lockfile this script owns when it is not sharing one. */
|
||||
ownLockKey: string;
|
||||
lock: string;
|
||||
/** The workspace dependency file whose lock this is, when it is one. */
|
||||
depFile: string | undefined;
|
||||
};
|
||||
|
||||
/** `f/foo.script.yaml` -> base `f/foo`, lock `f/foo.script.lock`;
|
||||
* `f/foo__mod/script.yaml` -> base `f/foo__mod/script`, lock `…/script.lock`.
|
||||
* The base is what the script's content file is named after. All returned
|
||||
* forward-slashed, whatever the map's separator. */
|
||||
function scriptMetaBase(
|
||||
key: string,
|
||||
): { base: string; ownLockKey: string; isJson: boolean } | undefined {
|
||||
const ref = toRefPath(key);
|
||||
for (const [suffix, isJson] of [
|
||||
[".script.yaml", false],
|
||||
[".script.json", true],
|
||||
["/script.yaml", false],
|
||||
["/script.json", true],
|
||||
] as const) {
|
||||
if (!ref.endsWith(suffix)) continue;
|
||||
const stripped = ref.slice(0, ref.length - suffix.length);
|
||||
if (suffix.startsWith("/")) {
|
||||
// `/script.yaml` is only script metadata inside a module folder; anywhere
|
||||
// else it is an ordinary file that happens to be called `script.yaml`.
|
||||
if (!stripped.endsWith("__mod")) return undefined;
|
||||
return {
|
||||
base: stripped + "/script",
|
||||
ownLockKey: stripped + "/script.lock",
|
||||
isJson,
|
||||
};
|
||||
}
|
||||
return { base: stripped, ownLockKey: stripped + ".script.lock", isJson };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Content-file extensions of the languages that carry a lock, longest first.
|
||||
* A language absent here is simply never deduplicated.
|
||||
* for related places search: ADD_NEW_LANG */
|
||||
const LOCKABLE_EXTS = [
|
||||
".fetch.ts",
|
||||
".deno.ts",
|
||||
".bun.ts",
|
||||
".playbook.yml",
|
||||
".ts",
|
||||
".py",
|
||||
".go",
|
||||
".php",
|
||||
".rs",
|
||||
];
|
||||
|
||||
/** Map keys grouped by directory, so a script's content file is looked up among
|
||||
* its own directory's entries: splitting a name at its first dot would put
|
||||
* `f/a.b.py` under `f/a` and leave every dotted script path undeduplicated. */
|
||||
function indexByDirectory(
|
||||
map: Record<string, string>,
|
||||
): Map<string, Set<string>> {
|
||||
const index = new Map<string, Set<string>>();
|
||||
for (const key of Object.keys(map)) {
|
||||
const ref = toRefPath(key);
|
||||
const dir = ref.slice(0, ref.lastIndexOf("/") + 1);
|
||||
const bucket = index.get(dir);
|
||||
if (bucket) {
|
||||
bucket.add(ref);
|
||||
} else {
|
||||
index.set(dir, new Set([ref]));
|
||||
}
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/** A script's content file and its language: `<base><ext>` and nothing looser,
|
||||
* since `f/a.b.py` is the content file of `f/a.b`, not of `f/a`. */
|
||||
function contentOfScript(
|
||||
map: Record<string, string>,
|
||||
base: string,
|
||||
byDirectory: Map<string, Set<string>>,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
): { content: string; language: ScriptLanguage } | undefined {
|
||||
const siblings = byDirectory.get(base.slice(0, base.lastIndexOf("/") + 1));
|
||||
if (!siblings) return undefined;
|
||||
for (const ext of LOCKABLE_EXTS) {
|
||||
const candidate = base + ext;
|
||||
if (!siblings.has(candidate)) continue;
|
||||
const content = map[toMapKey(candidate)] ?? map[candidate];
|
||||
if (content === undefined) continue;
|
||||
try {
|
||||
return {
|
||||
content,
|
||||
language: inferContentTypeFromFilePath(candidate, defaultTs),
|
||||
};
|
||||
} catch {
|
||||
// not a language this CLI knows
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** A map entry addressed by a forward-slashed path, whatever separator the map
|
||||
* was keyed with. */
|
||||
function lookup(
|
||||
map: Record<string, string>,
|
||||
refPath: string,
|
||||
): { key: string; content: string } | undefined {
|
||||
for (const key of [toMapKey(refPath), refPath]) {
|
||||
const content = map[key];
|
||||
if (content !== undefined) return { key, content };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Both parsers, declared format first. YAML is a superset of JSON, and
|
||||
* flow-style YAML (`{summary: x, lock: '!inline …'}`) starts with `{` while
|
||||
* failing `JSON.parse` — deciding by the first character loses the reference.
|
||||
*/
|
||||
function parseMetadata(
|
||||
metaPath: string,
|
||||
metaContent: string,
|
||||
isJson: boolean,
|
||||
): Record<string, any> | undefined {
|
||||
for (const asJson of isJson ? [true, false] : [false, true]) {
|
||||
try {
|
||||
const parsed = asJson
|
||||
? JSON.parse(metaContent)
|
||||
: yamlParseContent(metaPath, metaContent);
|
||||
if (typeof parsed === "object" && parsed !== null) return parsed;
|
||||
} catch {
|
||||
// try the other one
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The shared lockfile a metadata file's `lock` field names, if any. The raw text
|
||||
* is only a prefilter: a summary or a comment can carry the same words, and
|
||||
* `!inline` decides where a lock is written, so it is read from the parsed field
|
||||
* and nowhere else.
|
||||
*/
|
||||
export function sharedLockRefOf(
|
||||
metaPath: string,
|
||||
metaContent: string,
|
||||
isJson: boolean,
|
||||
): string | undefined {
|
||||
// Without the trailing space: the YAML serializer folds a long `lock:` line
|
||||
// at a space, so `!inline locks/…` can reach disk as `!inline\n locks/…`
|
||||
// and a prefilter looking for the space would call it a non-reader.
|
||||
if (!metaContent.includes(INLINE_PREFIX.trimEnd())) return undefined;
|
||||
const lock = parseMetadata(metaPath, metaContent, isJson)?.["lock"];
|
||||
if (typeof lock !== "string" || !lock.startsWith(INLINE_PREFIX)) {
|
||||
return undefined;
|
||||
}
|
||||
const ref = lock.slice(INLINE_PREFIX.length);
|
||||
return isSharedLockPath(ref) ? ref : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a metadata file may reference a shared lockfile but cannot say which.
|
||||
*
|
||||
* A `.script.yaml` carrying git conflict markers is a file whose `lock` cannot
|
||||
* be read, not one that reads nothing, and deleting a lockfile it may point at
|
||||
* is the unrecoverable half of that guess.
|
||||
*/
|
||||
export function metadataLockUnreadable(
|
||||
metaPath: string,
|
||||
metaContent: string,
|
||||
isJson: boolean,
|
||||
): boolean {
|
||||
if (!metaContent.includes(INLINE_PREFIX.trimEnd())) return false;
|
||||
return parseMetadata(metaPath, metaContent, isJson) === undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The shared lockfile a metadata FILE reads, when it reads one that is there.
|
||||
* `parseMetadataFile` resolves `lock` to the lockfile's content, so the
|
||||
* reference itself survives only in the raw text.
|
||||
*/
|
||||
export function sharedLockRefIn(
|
||||
metadataContent: string,
|
||||
isJson: boolean,
|
||||
root: string = ".",
|
||||
): string | undefined {
|
||||
const ref = sharedLockRefOf("metadata", metadataContent, isJson);
|
||||
return ref && existsSync(path.resolve(root, ref)) ? ref : undefined;
|
||||
}
|
||||
|
||||
/** The key a dependency file answers to, i.e. what a script names it by. */
|
||||
function depKeyOf(depFilePath: string): string | undefined {
|
||||
const info = workspaceDependenciesPathToLanguageAndFilename(depFilePath);
|
||||
return info && languageNeedsLock(info.language)
|
||||
? `${info.language} ${info.name ?? "default"}`
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Workspace dependency files keyed by the language and name a script names. */
|
||||
function depFilesByKey(paths: Iterable<string>): Map<string, string> {
|
||||
const byKey = new Map<string, string>();
|
||||
for (const key of paths) {
|
||||
const ref = toRefPath(key);
|
||||
if (!ref.startsWith("dependencies/")) continue;
|
||||
// A set named `team/python` exports as `dependencies/team/python.<file>`,
|
||||
// which has no distinct name under `locks/`: flattened it collides with the
|
||||
// top-level file, and the sweep would then retire a lockfile whose scripts
|
||||
// still read it. Such a set shares nothing and its scripts keep own locks.
|
||||
if (ref.slice("dependencies/".length).includes("/")) continue;
|
||||
const depKey = depKeyOf(ref);
|
||||
if (depKey) byKey.set(depKey, ref);
|
||||
}
|
||||
return byKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* The workspace dependency file whose lock a script's lock IS — undefined when
|
||||
* the script's lock is its own (see the header for the two cases).
|
||||
*/
|
||||
function shareableDepFile(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
depFiles: Map<string, string>,
|
||||
): string | undefined {
|
||||
// A script the worker locks differently for reasons of its own — a pinned
|
||||
// interpreter, `//npm`, `//nobundling` — cannot stand for its dependency
|
||||
// file's lock, so it never joins a group and its lock stays its own.
|
||||
if (hasLockAffectingAnnotation(scriptContent, language)) return undefined;
|
||||
const annotation = extractWorkspaceDepsAnnotation(scriptContent, language);
|
||||
if (annotation && (annotation.mode === "extra" || annotation.inline)) {
|
||||
return undefined;
|
||||
}
|
||||
const names = annotation ? annotation.external : ["default"];
|
||||
if (names.length !== 1) return undefined;
|
||||
return depFiles.get(`${language} ${names[0]}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* The shared lockfile a script belongs in, given the workspace dependency files
|
||||
* available — the one place that decides it, for both the sync planner and the
|
||||
* per-script regeneration in `updateScriptLock`.
|
||||
*/
|
||||
export function sharedLockTargetFor(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
depPaths: Iterable<string>,
|
||||
): string | undefined {
|
||||
const depFile = shareableDepFile(
|
||||
scriptContent,
|
||||
language,
|
||||
depFilesByKey(depPaths),
|
||||
);
|
||||
return depFile === undefined ? undefined : sharedLockPathFor(depFile);
|
||||
}
|
||||
|
||||
function collectScripts(
|
||||
map: Record<string, string>,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
depFiles: Map<string, string>,
|
||||
): ScriptEntry[] {
|
||||
const byDirectory = indexByDirectory(map);
|
||||
const entries: ScriptEntry[] = [];
|
||||
for (const [metaKey, metaContent] of Object.entries(map)) {
|
||||
const meta = scriptMetaBase(metaKey);
|
||||
if (!meta) continue;
|
||||
|
||||
const parsed = parseMetadata(metaKey, metaContent, meta.isJson);
|
||||
if (parsed === undefined) continue;
|
||||
|
||||
const lockRef = parsed["lock"];
|
||||
if (typeof lockRef !== "string" || !lockRef.startsWith(INLINE_PREFIX)) {
|
||||
continue;
|
||||
}
|
||||
const lockFile = lookup(map, lockRef.slice(INLINE_PREFIX.length));
|
||||
// An absent or empty lock is not a lock to share: a script with no
|
||||
// dependencies carries `lock: ''` and no file at all.
|
||||
if (lockFile === undefined || lockFile.content === "") continue;
|
||||
|
||||
const script = contentOfScript(map, meta.base, byDirectory, defaultTs);
|
||||
if (script === undefined || !languageNeedsLock(script.language)) continue;
|
||||
|
||||
entries.push({
|
||||
metaKey,
|
||||
isJson: meta.isJson,
|
||||
compactJson: !metaContent.includes("\n"),
|
||||
parsed,
|
||||
lockKey: lockFile.key,
|
||||
ownLockKey: toMapKey(meta.ownLockKey),
|
||||
lock: lockFile.content,
|
||||
depFile: shareableDepFile(script.content, script.language, depFiles),
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function serializeMetadata(entry: ScriptEntry): string {
|
||||
if (!entry.isJson) return yamlStringify(entry.parsed, yamlOptions);
|
||||
// Indented or compact as it was found: `sync` writes JSON metadata indented
|
||||
// and `generate-metadata` writes it compact, so imposing either one here
|
||||
// reformats files this feature exists to keep quiet.
|
||||
return entry.compactJson
|
||||
? JSON.stringify(entry.parsed)
|
||||
: JSON.stringify(entry.parsed, null, 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* What a sync map (path -> content) has to change for the scripts of a workspace
|
||||
* dependency file to share one lockfile. Pure: the map is not touched.
|
||||
*/
|
||||
export type SharedLockPlanContext = {
|
||||
defaultTs?: "bun" | "deno" | undefined;
|
||||
/**
|
||||
* Workspace dependency files to consider beyond the ones in `map`, for the
|
||||
* one caller whose map cannot hold them: `--skip-workspace-dependencies`.
|
||||
* Pass nothing otherwise — with dependency files in the map, an absence there
|
||||
* is a deletion, and adding disk's copy would keep a lockfile alive one sync
|
||||
* past the file it is named after.
|
||||
*/
|
||||
depFiles?: Iterable<string>;
|
||||
/**
|
||||
* Shared lockfiles the working tree already holds. The remote never
|
||||
* serializes one, so without this a sync that has no script for a dependency
|
||||
* file reads its lockfile as deleted — and every script still pointing at it
|
||||
* is left with an `!inline` that resolves to nothing.
|
||||
*/
|
||||
present?: Record<string, string>;
|
||||
};
|
||||
|
||||
export function computeSharedLockPlan(
|
||||
map: Record<string, string>,
|
||||
ctx: SharedLockPlanContext = {},
|
||||
): SharedLockPlan {
|
||||
const plan: SharedLockPlan = { writes: {}, deletes: [] };
|
||||
const depFiles = depFilesByKey([...Object.keys(map), ...(ctx.depFiles ?? [])]);
|
||||
// Every shared lockfile this sync can see, from either side.
|
||||
const present: Record<string, string> = { ...ctx.present };
|
||||
for (const [key, content] of Object.entries(map)) {
|
||||
if (isSharedLockPath(toRefPath(key))) present[toRefPath(key)] = content;
|
||||
}
|
||||
|
||||
const byDepFile = new Map<string, ScriptEntry[]>();
|
||||
const ownLock: ScriptEntry[] = [];
|
||||
for (const entry of collectScripts(map, ctx.defaultTs, depFiles)) {
|
||||
if (entry.depFile === undefined) {
|
||||
ownLock.push(entry);
|
||||
continue;
|
||||
}
|
||||
// Push into the existing array rather than rebuild it: a workspace where
|
||||
// every script shares one dependency file is the case this exists for, and
|
||||
// copying the group per insert makes that quadratic.
|
||||
const group = byDepFile.get(entry.depFile);
|
||||
if (group) group.push(entry);
|
||||
else byDepFile.set(entry.depFile, [entry]);
|
||||
}
|
||||
|
||||
const point = (entry: ScriptEntry, targetKey: string) => {
|
||||
if (entry.lockKey === targetKey) return;
|
||||
// A shared lockfile is dropped by the sweep below, which knows whether its
|
||||
// dependency file is still there; only a private one goes with its script.
|
||||
if (!isSharedLockPath(entry.lockKey)) plan.deletes.push(entry.lockKey);
|
||||
entry.parsed["lock"] = INLINE_PREFIX + toRefPath(targetKey);
|
||||
plan.writes[entry.metaKey] = serializeMetadata(entry);
|
||||
};
|
||||
|
||||
const takeOwnLock = (entry: ScriptEntry) => {
|
||||
if (map[entry.ownLockKey] !== entry.lock) {
|
||||
plan.writes[entry.ownLockKey] = entry.lock;
|
||||
}
|
||||
point(entry, entry.ownLockKey);
|
||||
};
|
||||
|
||||
for (const [depFile, group] of byDepFile) {
|
||||
// Every script here resolves against the same file and carries nothing the
|
||||
// worker locks separately, so their locks agree — unless one's committed
|
||||
// lock is simply behind. The many outvote the one; ties break on the content
|
||||
// itself so the outcome never depends on map ordering.
|
||||
const byContent = new Map<string, ScriptEntry[]>();
|
||||
for (const entry of group) {
|
||||
const sameLock = byContent.get(entry.lock);
|
||||
if (sameLock) sameLock.push(entry);
|
||||
else byContent.set(entry.lock, [entry]);
|
||||
}
|
||||
let content = "";
|
||||
let count = 0;
|
||||
for (const [lock, members] of byContent) {
|
||||
if (
|
||||
members.length > count ||
|
||||
(members.length === count && lock < content)
|
||||
) {
|
||||
content = lock;
|
||||
count = members.length;
|
||||
}
|
||||
}
|
||||
|
||||
const sharedKey = toMapKey(sharedLockPathFor(depFile));
|
||||
if (map[sharedKey] !== content) plan.writes[sharedKey] = content;
|
||||
for (const entry of group) {
|
||||
if (entry.lock === content) point(entry, sharedKey);
|
||||
else takeOwnLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
// A script that stopped resolving against a dependency file takes its lock
|
||||
// back with it.
|
||||
for (const entry of ownLock) {
|
||||
if (entry.lockKey !== entry.ownLockKey) takeOwnLock(entry);
|
||||
}
|
||||
|
||||
// A shared lockfile lives exactly as long as the dependency file it is named
|
||||
// after. Asking that, rather than "does any script still read it", is what
|
||||
// lets a sync narrowed to one item leave the rest of the workspace alone: a
|
||||
// lockfile with no script in view is carried forward, not deleted.
|
||||
for (const [sharedRef, content] of Object.entries(present)) {
|
||||
const depFile = depFileOfSharedLock(sharedRef);
|
||||
if (depFile === undefined) continue;
|
||||
const key = toMapKey(sharedRef);
|
||||
if (plan.writes[key] !== undefined) continue;
|
||||
if (depFiles.has(depKeyOf(depFile) ?? "")) {
|
||||
if (map[key] === undefined) plan.writes[key] = content;
|
||||
} else {
|
||||
plan.deletes.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Deletes are applied after writes, so a path some script still writes must
|
||||
// not also be dropped — two metadata files pointing at one lock file would
|
||||
// otherwise cancel each other out and leave the survivor without a lock.
|
||||
plan.deletes = plan.deletes.filter((key) => plan.writes[key] === undefined);
|
||||
|
||||
return plan;
|
||||
}
|
||||
|
||||
export function applySharedLockPlanToMap(
|
||||
map: Record<string, string>,
|
||||
plan: SharedLockPlan,
|
||||
): void {
|
||||
for (const [key, content] of Object.entries(plan.writes)) {
|
||||
map[key] = content;
|
||||
}
|
||||
for (const key of plan.deletes) {
|
||||
delete map[key];
|
||||
}
|
||||
}
|
||||
|
||||
export async function applySharedLockPlanToDisk(
|
||||
plan: SharedLockPlan,
|
||||
): Promise<void> {
|
||||
for (const [key, content] of Object.entries(plan.writes)) {
|
||||
// Per write, so `locks/` comes into existence only when there is a shared
|
||||
// lockfile to put in it.
|
||||
await mkdir(path.dirname(key), { recursive: true });
|
||||
await writeFile(key, content, "utf-8");
|
||||
}
|
||||
for (const key of plan.deletes) {
|
||||
await rm(key, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The script metadata files that read a given shared lockfile. A change to that
|
||||
* file is a change to their lock, and they are what carries it to the remote.
|
||||
*/
|
||||
export function scriptsReferencingSharedLock(
|
||||
map: Record<string, string>,
|
||||
sharedKey: string,
|
||||
): string[] {
|
||||
const reference = toRefPath(sharedKey);
|
||||
const referrers: string[] = [];
|
||||
for (const [metaKey, metaContent] of Object.entries(map)) {
|
||||
const meta = scriptMetaBase(metaKey);
|
||||
if (!meta) continue;
|
||||
if (sharedLockRefOf(metaKey, metaContent, meta.isJson) === reference) {
|
||||
referrers.push(metaKey);
|
||||
}
|
||||
}
|
||||
return referrers;
|
||||
}
|
||||
+50
-126
@@ -4,7 +4,7 @@ import { colors } from "@cliffy/ansi/colors";
|
||||
import * as log from "../core/log.ts";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { yamlParseFile } from "./yaml.ts";
|
||||
import { writeFile, stat, rm, readdir } from "node:fs/promises";
|
||||
import { writeFile, stat, rm, readdir, mkdir } from "node:fs/promises";
|
||||
import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
@@ -17,8 +17,20 @@ import {
|
||||
ScriptLanguage,
|
||||
workspaceDependenciesLanguages,
|
||||
languageNeedsLock,
|
||||
LANG_COMMENT_LIT,
|
||||
} from "./script_common.ts";
|
||||
import { inferContentTypeFromFilePath } from "./script_common.ts";
|
||||
// Workspace-dependency vocabulary lives with the languages it describes; these
|
||||
// re-exports keep the CLI's existing import sites working.
|
||||
export {
|
||||
workspaceDependenciesPathToLanguageAndFilename,
|
||||
extractWorkspaceDepsAnnotation,
|
||||
type WorkspaceDepsAnnotation,
|
||||
} from "./script_common.ts";
|
||||
import {
|
||||
workspaceDependenciesPathToLanguageAndFilename,
|
||||
extractWorkspaceDepsAnnotation,
|
||||
} from "./script_common.ts";
|
||||
import { dbtGeneratedDirs, isUnderGeneratedDir, isBundledModuleFile, getModuleFolderSuffix, isModuleEntryPoint, scriptPathToRemotePath } from "./resource_folders.ts";
|
||||
import { findCodebase, yamlOptions } from "../commands/sync/sync.ts";
|
||||
import { generateHash, readInlinePathSync, getHeaders, readTextFile, readTextFileSync } from "./utils.ts";
|
||||
@@ -31,6 +43,7 @@ import { getIsWin } from "./utils.ts";
|
||||
import { extractRelativeImports } from "./relative_imports.ts";
|
||||
import { DoubleLinkedDependencyTree } from "./dependency_tree.ts";
|
||||
import { pollJobWithQueueLogging } from "./job_polling.ts";
|
||||
import { sharedLockRefIn, sharedLockTargetFor } from "./lock_dedup.ts";
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
const _parserCache = new Map<string, Promise<any>>();
|
||||
@@ -106,17 +119,6 @@ export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Pro
|
||||
return rawWorkspaceDeps;
|
||||
}
|
||||
|
||||
export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined {
|
||||
const relativePath = path.replace("dependencies/", "");
|
||||
for (const { filename, language } of workspaceDependenciesLanguages) {
|
||||
if (relativePath.endsWith(filename)) {
|
||||
return {
|
||||
name: relativePath === filename ? undefined : relativePath.replace("." + filename, ""),
|
||||
language
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters raw workspace dependencies to only include those that:
|
||||
@@ -203,6 +205,7 @@ export async function generateScriptMetadataInternal(
|
||||
schemaOnly?: boolean | undefined;
|
||||
defaultTs?: "bun" | "deno";
|
||||
rehashOnly?: boolean | undefined;
|
||||
dedupeLockfiles?: boolean | undefined;
|
||||
},
|
||||
dryRun: boolean,
|
||||
noStaleMessage: boolean,
|
||||
@@ -375,6 +378,15 @@ export async function generateScriptMetadataInternal(
|
||||
filteredRawWorkspaceDependencies,
|
||||
tempScriptRefs,
|
||||
lockPathOverride,
|
||||
// The lockfile of the workspace dependency file this script resolves
|
||||
// against, if any: joining it is a matter of resolving to its content.
|
||||
opts.dedupeLockfiles
|
||||
? sharedLockTargetFor(
|
||||
scriptContent,
|
||||
language,
|
||||
Object.keys(rawWorkspaceDependencies),
|
||||
)
|
||||
: undefined,
|
||||
);
|
||||
} else {
|
||||
metadataParsedContent.lock = "";
|
||||
@@ -406,7 +418,14 @@ export async function generateScriptMetadataInternal(
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if (metadataInFolder) {
|
||||
// `parseMetadataFile` resolved `lock` to the lockfile's CONTENT, so the
|
||||
// reference has to be restored from the raw text — including a shared one
|
||||
// (`dedupeLockfiles`), which `--schema-only` would otherwise replace with a
|
||||
// per-script path whose file deduplication removed.
|
||||
const sharedRef = sharedLockRefIn(metadataContent, metadataWithType.isJson);
|
||||
if (sharedRef) {
|
||||
metadataParsedContent.lock = "!inline " + sharedRef;
|
||||
} else if (metadataInFolder) {
|
||||
metadataParsedContent.lock =
|
||||
"!inline " + remotePath.replaceAll(SEP, "/") + getModuleFolderSuffix() + "/script.lock";
|
||||
} else {
|
||||
@@ -493,119 +512,6 @@ export async function updateScriptSchema(
|
||||
delete metadataContent.no_main_func;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Annotation parser — mirrors backend's WorkspaceDependenciesAnnotatedRefs::parse
|
||||
// (windmill-common/src/workspace_dependencies.rs) so the cache key captures
|
||||
// exactly the parts of scriptContent that affect lockfile generation.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type AnnotationMode = "manual" | "extra";
|
||||
|
||||
interface WorkspaceDepsAnnotation {
|
||||
mode: AnnotationMode;
|
||||
external: string[];
|
||||
inline: string | null;
|
||||
}
|
||||
|
||||
const LANG_ANNOTATION_CONFIG: Partial<
|
||||
Record<ScriptLanguage, { comment: string; keyword: string; validityRe?: RegExp }>
|
||||
> = {
|
||||
python3: { comment: "#", keyword: "requirements", validityRe: /^#\s?(\S+)\s*$/ },
|
||||
bun: { comment: "//", keyword: "package_json" },
|
||||
nativets: { comment: "//", keyword: "package_json" },
|
||||
go: { comment: "//", keyword: "go_mod" },
|
||||
php: { comment: "//", keyword: "composer_json" },
|
||||
powershell: { comment: "#", keyword: "modules_json" },
|
||||
};
|
||||
|
||||
export function extractWorkspaceDepsAnnotation(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
): WorkspaceDepsAnnotation | null {
|
||||
const config = LANG_ANNOTATION_CONFIG[language];
|
||||
if (!config) return null;
|
||||
|
||||
const { comment, keyword, validityRe } = config;
|
||||
const extraMarkerUnderscore = `extra_${keyword}:`;
|
||||
const extraMarkerHyphen = `extra-${keyword}:`;
|
||||
const manualMarker = `${keyword}:`;
|
||||
|
||||
const stripComment = (l: string): string | null => {
|
||||
if (!l.startsWith(comment)) return null;
|
||||
return l.substring(comment.length).trimStart();
|
||||
};
|
||||
const isExtra = (l: string): boolean => {
|
||||
const s = stripComment(l);
|
||||
return s !== null && (s.startsWith(extraMarkerUnderscore) || s.startsWith(extraMarkerHyphen));
|
||||
};
|
||||
const isManual = (l: string): boolean => {
|
||||
const s = stripComment(l);
|
||||
return s !== null && s.startsWith(manualMarker);
|
||||
};
|
||||
|
||||
const lines = scriptContent.split("\n");
|
||||
|
||||
// Find first annotation line (mirrors Rust find_position)
|
||||
let pos = -1;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (isExtra(lines[i]) || isManual(lines[i])) {
|
||||
pos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (pos === -1) return null;
|
||||
|
||||
const annotationLine = lines[pos];
|
||||
const mode: AnnotationMode = isExtra(annotationLine) ? "extra" : "manual";
|
||||
|
||||
// Parse external references from the annotation line
|
||||
const marker = mode === "extra"
|
||||
? (annotationLine.includes(extraMarkerUnderscore) ? extraMarkerUnderscore : extraMarkerHyphen)
|
||||
: manualMarker;
|
||||
const unparsed = annotationLine.replaceAll(marker, "").replaceAll(comment, "");
|
||||
const external = unparsed
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
// Parse inline deps from subsequent lines
|
||||
const inlineParts: string[] = [];
|
||||
for (let i = pos + 1; i < lines.length; i++) {
|
||||
const l = lines[i];
|
||||
if (validityRe) {
|
||||
const match = validityRe.exec(l);
|
||||
if (match && match[1]) {
|
||||
inlineParts.push(match[1]);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (!l.startsWith(comment)) {
|
||||
break;
|
||||
}
|
||||
inlineParts.push(l.substring(comment.length));
|
||||
}
|
||||
}
|
||||
|
||||
const inlineStr = inlineParts.join("\n");
|
||||
const inline = inlineStr.trim().length > 0 ? inlineStr : null;
|
||||
|
||||
return { mode, external, inline };
|
||||
}
|
||||
|
||||
// Mirrors backend ScriptLang::as_comment_lit (windmill-types/src/scripts.rs)
|
||||
// for the languages that can reach the lock cache.
|
||||
const LANG_COMMENT_LIT: Partial<Record<ScriptLanguage, string>> = {
|
||||
python3: "#",
|
||||
ansible: "#",
|
||||
powershell: "#",
|
||||
bun: "//",
|
||||
nativets: "//",
|
||||
deno: "//",
|
||||
go: "//",
|
||||
php: "//",
|
||||
rust: "//!",
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the leading comment/blank-line block of the script, verbatim.
|
||||
@@ -783,6 +689,7 @@ async function updateScriptLock(
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>,
|
||||
lockPathOverride?: string,
|
||||
sharedLockRef?: string,
|
||||
): Promise<void> {
|
||||
if (!languageNeedsLock(language)) {
|
||||
// A dbt lock is written by the dependency job on a worker, from a real
|
||||
@@ -816,6 +723,23 @@ async function updateScriptLock(
|
||||
|
||||
const lockPath = lockPathOverride ?? remotePath + ".script.lock";
|
||||
if (lock != "") {
|
||||
// Joins an agreeing shared lockfile, and never creates or moves one: this
|
||||
// runs per script and in parallel, so two scripts that resolve differently
|
||||
// — a pinned Python version, an `//npm` annotation — would both find the
|
||||
// file absent, both write it, and one would lose its lock with no copy left
|
||||
// anywhere. Deciding a shared lockfile's content is the whole-tree pass's
|
||||
// job, which sees every script at once.
|
||||
if (
|
||||
sharedLockRef &&
|
||||
existsSync(sharedLockRef) &&
|
||||
readTextFileSync(sharedLockRef) === lock
|
||||
) {
|
||||
if (existsSync(lockPath)) {
|
||||
await rm(lockPath);
|
||||
}
|
||||
metadataContent.lock = "!inline " + sharedLockRef;
|
||||
return;
|
||||
}
|
||||
await writeFile(lockPath, lock, "utf-8");
|
||||
metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
|
||||
} else {
|
||||
|
||||
@@ -44,6 +44,222 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
|
||||
{ language: "powershell", filename: "modules.json" },
|
||||
] as const;
|
||||
|
||||
export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined {
|
||||
const relativePath = path.replace("dependencies/", "");
|
||||
for (const { filename, language } of workspaceDependenciesLanguages) {
|
||||
if (relativePath.endsWith(filename)) {
|
||||
return {
|
||||
name: relativePath === filename ? undefined : relativePath.replace("." + filename, ""),
|
||||
language
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Annotation parser — mirrors backend's WorkspaceDependenciesAnnotatedRefs::parse
|
||||
// (windmill-common/src/workspace_dependencies.rs), so the CLI can tell which
|
||||
// workspace dependency file a script resolves against without asking a worker.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type AnnotationMode = "manual" | "extra";
|
||||
|
||||
export interface WorkspaceDepsAnnotation {
|
||||
mode: AnnotationMode;
|
||||
external: string[];
|
||||
inline: string | null;
|
||||
}
|
||||
|
||||
const LANG_ANNOTATION_CONFIG: Partial<
|
||||
Record<ScriptLanguage, { comment: string; keyword: string; validityRe?: RegExp }>
|
||||
> = {
|
||||
python3: { comment: "#", keyword: "requirements", validityRe: /^#\s?(\S+)\s*$/ },
|
||||
bun: { comment: "//", keyword: "package_json" },
|
||||
nativets: { comment: "//", keyword: "package_json" },
|
||||
go: { comment: "//", keyword: "go_mod" },
|
||||
php: { comment: "//", keyword: "composer_json" },
|
||||
powershell: { comment: "#", keyword: "modules_json" },
|
||||
};
|
||||
|
||||
export function extractWorkspaceDepsAnnotation(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
): WorkspaceDepsAnnotation | null {
|
||||
const config = LANG_ANNOTATION_CONFIG[language];
|
||||
if (!config) return null;
|
||||
|
||||
const { comment, keyword, validityRe } = config;
|
||||
const extraMarkerUnderscore = `extra_${keyword}:`;
|
||||
const extraMarkerHyphen = `extra-${keyword}:`;
|
||||
const manualMarker = `${keyword}:`;
|
||||
|
||||
const stripComment = (l: string): string | null => {
|
||||
if (!l.startsWith(comment)) return null;
|
||||
return l.substring(comment.length).trimStart();
|
||||
};
|
||||
const isExtra = (l: string): boolean => {
|
||||
const s = stripComment(l);
|
||||
return s !== null && (s.startsWith(extraMarkerUnderscore) || s.startsWith(extraMarkerHyphen));
|
||||
};
|
||||
const isManual = (l: string): boolean => {
|
||||
const s = stripComment(l);
|
||||
return s !== null && s.startsWith(manualMarker);
|
||||
};
|
||||
|
||||
const lines = scriptContent.split("\n");
|
||||
|
||||
// Find first annotation line (mirrors Rust find_position)
|
||||
let pos = -1;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (isExtra(lines[i]) || isManual(lines[i])) {
|
||||
pos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (pos === -1) return null;
|
||||
|
||||
const annotationLine = lines[pos];
|
||||
const mode: AnnotationMode = isExtra(annotationLine) ? "extra" : "manual";
|
||||
|
||||
// Parse external references from the annotation line
|
||||
const marker = mode === "extra"
|
||||
? (annotationLine.includes(extraMarkerUnderscore) ? extraMarkerUnderscore : extraMarkerHyphen)
|
||||
: manualMarker;
|
||||
const unparsed = annotationLine.replaceAll(marker, "").replaceAll(comment, "");
|
||||
const external = unparsed
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
// Parse inline deps from subsequent lines
|
||||
const inlineParts: string[] = [];
|
||||
for (let i = pos + 1; i < lines.length; i++) {
|
||||
const l = lines[i];
|
||||
if (validityRe) {
|
||||
const match = validityRe.exec(l);
|
||||
if (match && match[1]) {
|
||||
inlineParts.push(match[1]);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (!l.startsWith(comment)) {
|
||||
break;
|
||||
}
|
||||
inlineParts.push(l.substring(comment.length));
|
||||
}
|
||||
}
|
||||
|
||||
const inlineStr = inlineParts.join("\n");
|
||||
const inline = inlineStr.trim().length > 0 ? inlineStr : null;
|
||||
|
||||
return { mode, external, inline };
|
||||
}
|
||||
|
||||
/** The comment marker each language's annotations are written behind. */
|
||||
export const LANG_COMMENT_LIT: Partial<Record<ScriptLanguage, string>> = {
|
||||
python3: "#",
|
||||
ansible: "#",
|
||||
powershell: "#",
|
||||
bun: "//",
|
||||
nativets: "//",
|
||||
deno: "//",
|
||||
go: "//",
|
||||
php: "//",
|
||||
rust: "//!",
|
||||
};
|
||||
|
||||
/**
|
||||
* The annotations each language recognises, by the exact names the worker
|
||||
* matches (`#[annotations(..)]` structs in windmill-common/src/worker.rs).
|
||||
* Several change what it locks — a pinned interpreter, `npm`, `nobundling` —
|
||||
* and the rest are cheap to treat the same way, since the only cost is that
|
||||
* such a script keeps a lockfile of its own.
|
||||
* for related places search: ADD_NEW_LANG
|
||||
*/
|
||||
const LANG_ANNOTATIONS: Partial<Record<ScriptLanguage, string[]>> = {
|
||||
python3: [
|
||||
"no_cache",
|
||||
"no_postinstall",
|
||||
"py_select_latest",
|
||||
"skip_result_postprocessing",
|
||||
"py310",
|
||||
"py311",
|
||||
"py312",
|
||||
"py313",
|
||||
"sandbox",
|
||||
],
|
||||
bun: ["npm", "nodejs", "native", "nobundling", "sandbox"],
|
||||
nativets: ["npm", "nodejs", "native", "nobundling", "sandbox"],
|
||||
deno: ["npm", "nodejs", "native", "nobundling", "sandbox"],
|
||||
go: ["go1_22_compat"],
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether a script's leading comment block carries an annotation the worker
|
||||
* acts on, which means its lock may not be its dependency file's.
|
||||
*
|
||||
* Matched the way the worker matches: the key is the line, or what precedes the
|
||||
* first `=`, and it has to BE one of the names above. Unknown keys are ignored
|
||||
* there and so here — which is what keeps `# TODO:` or `# type: ignore` from
|
||||
* quietly dropping an ordinary documented script out of deduplication.
|
||||
*
|
||||
* `# py: <specifier>` is the exception the macro does not cover: the python
|
||||
* import parser reads it directly (`windmill-parser-py-imports`, alongside the
|
||||
* `py310`..`py313` flags) to pick the interpreter, which changes what resolves.
|
||||
*/
|
||||
export function hasLockAffectingAnnotation(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
): boolean {
|
||||
const comment = LANG_COMMENT_LIT[language];
|
||||
const names = LANG_ANNOTATIONS[language];
|
||||
if (!comment || !names) return false;
|
||||
for (const line of scriptContent.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === "") continue;
|
||||
if (!trimmed.startsWith(comment)) break; // past the header block
|
||||
// Matched on the raw line: the parser tests `# py:`/`#py:` before trimming.
|
||||
if (language === "python3" && /^#\s?py:/.test(line)) return true;
|
||||
const body = trimmed.slice(comment.length).trim();
|
||||
const key = body.split("=")[0].trim();
|
||||
if (names.includes(key)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Where the lockfiles shared by several scripts live when `dedupeLockfiles`
|
||||
* is on — see `utils/lock_dedup.ts`. A top-level directory of its own: what a
|
||||
* group shares is a resolved lock, which needs no workspace dependency file
|
||||
* behind it, and inline-script locks would belong here too. */
|
||||
export const SHARED_LOCK_DIR = "locks";
|
||||
|
||||
/** The lockfile shared by the scripts that resolve against a workspace
|
||||
* dependency file: its own name, plus `.lock`. Appending rather than replacing
|
||||
* the extension keeps the correspondence exact and reversible —
|
||||
* `dependencies/team_a.requirements.in` <-> `locks/team_a.requirements.in.lock`. */
|
||||
export function sharedLockPathFor(depFilePath: string): string {
|
||||
const name = depFilePath.replaceAll("\\", "/").split("/").pop()!;
|
||||
return `${SHARED_LOCK_DIR}/${name}.lock`;
|
||||
}
|
||||
|
||||
/** The workspace dependency file a shared lockfile belongs to, if it is one. */
|
||||
export function depFileOfSharedLock(p: string): string | undefined {
|
||||
const normalized = p.replaceAll("\\", "/");
|
||||
if (!normalized.startsWith(SHARED_LOCK_DIR + "/")) return undefined;
|
||||
const name = normalized.slice(SHARED_LOCK_DIR.length + 1);
|
||||
if (name.includes("/") || !name.endsWith(".lock")) return undefined;
|
||||
const depFile = "dependencies/" + name.slice(0, -".lock".length);
|
||||
const info = workspaceDependenciesPathToLanguageAndFilename(depFile);
|
||||
// `locks/vendor.lock` names no dependency file, so it is not Windmill's: a
|
||||
// repo that already keeps lockfiles here keeps them.
|
||||
return info && languageNeedsLock(info.language) ? depFile : undefined;
|
||||
}
|
||||
|
||||
export function isSharedLockPath(p: string): boolean {
|
||||
return depFileOfSharedLock(p) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if a script in the given language requires a lock file.
|
||||
* Matches the condition in updateScriptLock (metadata.ts).
|
||||
|
||||
Reference in New Issue
Block a user