mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-20 00:02:19 +00:00
Ruben Fiszel
4abe589397
Add a --locks-required flag that fails if scripts or inline scripts that need locks have no locks. Checks standalone scripts, flow inline scripts, app inline scripts, and raw app backend scripts. The flag can be set via CLI (--locks-required) or wmill.yaml config (locksRequired: true). On sync push, verification runs before any push operations to fail early. Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
3120 lines
96 KiB
TypeScript
3120 lines
96 KiB
TypeScript
import { requireLogin } from "../../core/auth.ts";
|
|
import { fetchVersion, resolveWorkspace } from "../../core/context.ts";
|
|
import {
|
|
colors,
|
|
Command,
|
|
Confirm,
|
|
ensureDir,
|
|
JSZip,
|
|
log,
|
|
minimatch,
|
|
path,
|
|
SEP,
|
|
yamlParseContent,
|
|
yamlStringify,
|
|
} from "../../../deps.ts";
|
|
import * as wmill from "../../../gen/services.gen.ts";
|
|
|
|
import {
|
|
getTypeStrFromPath,
|
|
GlobalOptions,
|
|
parseFromPath,
|
|
pushObj,
|
|
showConflict,
|
|
showDiff,
|
|
extractNativeTriggerInfo,
|
|
} from "../../types.ts";
|
|
import { downloadZip } from "./pull.ts";
|
|
import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts";
|
|
|
|
import {
|
|
exts,
|
|
findContentFile,
|
|
findResourceFile,
|
|
handleScriptMetadata,
|
|
removeExtensionToPath,
|
|
} from "../script/script.ts";
|
|
|
|
import { handleFile } from "../script/script.ts";
|
|
import {
|
|
deepEqual,
|
|
fetchRemoteVersion,
|
|
isFileResource,
|
|
isRawAppFile,
|
|
isWorkspaceDependencies,
|
|
} from "../../utils/utils.ts";
|
|
import {
|
|
getEffectiveSettings,
|
|
mergeConfigWithConfigFile,
|
|
SyncOptions,
|
|
validateBranchConfiguration,
|
|
} from "../../core/conf.ts";
|
|
import {
|
|
fromBranchSpecificPath,
|
|
getBranchSpecificPath,
|
|
getSpecificItemsForCurrentBranch,
|
|
isBranchSpecificFile,
|
|
isCurrentBranchFile,
|
|
isItemTypeConfigured,
|
|
isSpecificItem,
|
|
SpecificItemsConfig,
|
|
} from "../../core/specific_items.ts";
|
|
import { getCurrentGitBranch, isGitRepository } from "../../utils/git.ts";
|
|
import { Workspace } from "../workspace/workspace.ts";
|
|
import { removePathPrefix } from "../../types.ts";
|
|
import { listSyncCodebases, SyncCodebase } from "../../utils/codebase.ts";
|
|
import {
|
|
generateScriptMetadataInternal,
|
|
getRawWorkspaceDependencies,
|
|
readLockfile,
|
|
workspaceDependenciesPathToLanguageAndFilename,
|
|
} from "../../utils/metadata.ts";
|
|
import { OpenFlow, NativeServiceName } from "../../../gen/types.gen.ts";
|
|
import { pushResource } from "../resource/resource.ts";
|
|
import {
|
|
newPathAssigner,
|
|
newRawAppPathAssigner,
|
|
PathAssigner,
|
|
} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
|
|
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
|
|
import { generateFlowLockInternal } from "../flow/flow_metadata.ts";
|
|
import { isExecutionModeAnonymous } from "../app/app.ts";
|
|
import {
|
|
APP_BACKEND_FOLDER,
|
|
generateAppLocksInternal,
|
|
} from "../app/app_metadata.ts";
|
|
import {
|
|
isFlowPath,
|
|
isAppPath,
|
|
isRawAppPath,
|
|
extractFolderPath,
|
|
extractResourceName,
|
|
isFlowMetadataFile,
|
|
isAppMetadataFile,
|
|
isRawAppMetadataFile,
|
|
isRawAppFolderMetadataFile,
|
|
getDeleteSuffix,
|
|
transformJsonPathToDir,
|
|
getFolderSuffix,
|
|
getFolderSuffixWithSep,
|
|
getNonDottedPaths,
|
|
} from "../../utils/resource_folders.ts";
|
|
|
|
// Merge CLI options with effective settings, preserving CLI flags as overrides
|
|
function mergeCliWithEffectiveOptions<
|
|
T extends GlobalOptions & SyncOptions & { repository?: string },
|
|
>(cliOpts: T, effectiveOpts: SyncOptions): T {
|
|
// overlay CLI options on top (undefined cliOpts won't override effectiveOpts)
|
|
return Object.assign({}, effectiveOpts, cliOpts) as T;
|
|
}
|
|
|
|
// Resolve effective sync options using branch-based configuration
|
|
async function resolveEffectiveSyncOptions(
|
|
workspace: Workspace,
|
|
localConfig: SyncOptions,
|
|
promotion?: string,
|
|
branchOverride?: string,
|
|
): Promise<SyncOptions> {
|
|
return await getEffectiveSettings(localConfig, promotion, false, false, branchOverride);
|
|
}
|
|
|
|
type DynFSElement = {
|
|
isDirectory: boolean;
|
|
path: string;
|
|
whitelistedExt?: boolean;
|
|
// getContentBytes(): Promise<Uint8Array>;
|
|
getContentText(): Promise<string>;
|
|
getChildren(): AsyncIterable<DynFSElement>;
|
|
};
|
|
|
|
export function findCodebase(
|
|
path: string,
|
|
codebases: SyncCodebase[],
|
|
): SyncCodebase | undefined {
|
|
if (!path.endsWith(".ts")) {
|
|
return;
|
|
}
|
|
for (const c of codebases) {
|
|
let included = false;
|
|
let excluded = false;
|
|
if (c.includes == undefined || c.includes == null) {
|
|
included = true;
|
|
}
|
|
if (typeof c.includes == "string") {
|
|
c.includes = [c.includes];
|
|
}
|
|
for (const r of c.includes ?? []) {
|
|
if (included) {
|
|
break;
|
|
}
|
|
if (minimatch(path, r)) {
|
|
included = true;
|
|
}
|
|
}
|
|
if (typeof c.excludes == "string") {
|
|
c.excludes = [c.excludes];
|
|
}
|
|
for (const r of c.excludes ?? []) {
|
|
if (minimatch(path, r)) {
|
|
excluded = true;
|
|
}
|
|
}
|
|
if (included && !excluded) {
|
|
return c;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function addCodebaseDigestIfRelevant(
|
|
path: string,
|
|
content: string,
|
|
codebases: SyncCodebase[],
|
|
ignoreCodebaseChanges: boolean,
|
|
): Promise<string> {
|
|
const isScript = path.endsWith(".script.yaml");
|
|
if (!isScript) {
|
|
return content;
|
|
}
|
|
let isTs = true;
|
|
const replacedPath = path.replace(".script.yaml", ".ts");
|
|
try {
|
|
await Deno.stat(replacedPath);
|
|
} catch {
|
|
isTs = false;
|
|
}
|
|
if (!isTs) {
|
|
return content;
|
|
}
|
|
if (isTs) {
|
|
const c = findCodebase(replacedPath, codebases);
|
|
if (c) {
|
|
let parsed: any;
|
|
try {
|
|
parsed = yamlParseContent(path, content);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to parse YAML content for codebase digest at path: ${path}`,
|
|
);
|
|
throw error;
|
|
}
|
|
if (parsed && typeof parsed == "object") {
|
|
if (ignoreCodebaseChanges) {
|
|
parsed["codebase"] = undefined;
|
|
} else {
|
|
parsed["codebase"] = await c.getDigest();
|
|
}
|
|
parsed["lock"] = "";
|
|
return yamlStringify(parsed, yamlOptions);
|
|
} else {
|
|
throw Error(
|
|
`Expected local yaml ${path} to be an object, found: ${content} instead`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
return content;
|
|
}
|
|
|
|
export async function FSFSElement(
|
|
p: string,
|
|
codebases: SyncCodebase[],
|
|
ignoreCodebaseChanges: boolean,
|
|
): Promise<DynFSElement> {
|
|
function _internal_element(
|
|
localP: string,
|
|
isDir: boolean,
|
|
codebases: SyncCodebase[],
|
|
): DynFSElement {
|
|
return {
|
|
isDirectory: isDir,
|
|
path: localP.substring(p.length + 1),
|
|
async *getChildren(): AsyncIterable<DynFSElement> {
|
|
if (!isDir) return [];
|
|
try {
|
|
for await (const e of Deno.readDir(localP)) {
|
|
yield _internal_element(
|
|
path.join(localP, e.name),
|
|
e.isDirectory,
|
|
codebases,
|
|
);
|
|
}
|
|
} catch (e) {
|
|
log.warn(`Error reading dir: ${localP}, ${e}`);
|
|
}
|
|
},
|
|
// async getContentBytes(): Promise<Uint8Array> {
|
|
// return await Deno.readFile(localP);
|
|
// },
|
|
async getContentText(): Promise<string> {
|
|
const content = await Deno.readTextFile(localP);
|
|
const itemPath = localP.substring(p.length + 1);
|
|
const r = await addCodebaseDigestIfRelevant(
|
|
itemPath,
|
|
content,
|
|
codebases,
|
|
ignoreCodebaseChanges,
|
|
);
|
|
return r;
|
|
},
|
|
};
|
|
}
|
|
return _internal_element(p, (await Deno.stat(p)).isDirectory, codebases);
|
|
}
|
|
|
|
function prioritizeName(name: string): string {
|
|
if (name == "version") return "aaa";
|
|
if (name == "id") return "aa";
|
|
if (name == "type") return "ab";
|
|
if (name == "summary") return "ad";
|
|
if (name == "name") return "ae";
|
|
if (name == "display_name") return "af";
|
|
if (name == "description") return "ag";
|
|
if (name == "value") return "ah";
|
|
if (name == "content") return "ai";
|
|
if (name == "modules") return "aj";
|
|
if (name == "failure_module") return "ak";
|
|
if (name == "input_transforms") return "al";
|
|
if (name == "lock") return "az";
|
|
if (name == "locks") return "azz";
|
|
|
|
return name;
|
|
}
|
|
|
|
export const yamlOptions = {
|
|
sortKeys: (a: any, b: any) => {
|
|
return prioritizeName(a).localeCompare(prioritizeName(b));
|
|
},
|
|
noCompatMode: true,
|
|
noRefs: true,
|
|
skipInvalid: true,
|
|
};
|
|
|
|
export interface InlineScript {
|
|
path: string;
|
|
content: string;
|
|
}
|
|
|
|
function extractFields(fields: Record<string, any>) {
|
|
Object.entries(fields).forEach(([k, v]) => {
|
|
if (typeof v == "object") {
|
|
if (v.type == "static") {
|
|
fields[k] = { value: v.value };
|
|
} else if (v.type == "javascript") {
|
|
fields[k] = { expr: v.expr, allowUserResources: v.allowUserResources };
|
|
} else if (v.type == "user") {
|
|
fields[k] = undefined;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
export function extractFieldsForRawApps(runnables: Record<string, any>) {
|
|
Object.values(runnables).forEach((v) => {
|
|
if (typeof v == "object") {
|
|
if (v.fields !== undefined) {
|
|
extractFields(v.fields);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Generates AGENTS.md - app-specific configuration for AI agents working with raw apps.
|
|
* References the raw-app skill for complete documentation and includes instance-specific
|
|
* data configuration (datatable, schema, whitelisted tables).
|
|
*/
|
|
export function generateAgentsDocumentation(data: {
|
|
tables?: string[];
|
|
datatable?: string;
|
|
schema?: string;
|
|
} | undefined): string {
|
|
const tables = data?.tables ?? [];
|
|
const defaultDatatable = data?.datatable;
|
|
const defaultSchema = data?.schema;
|
|
|
|
return `# AI Agent Instructions
|
|
|
|
For complete raw app documentation (app structure, backend runnables, datatables, SQL migrations), use the \`raw-app\` skill.
|
|
|
|
This file contains **app-specific configuration** for this raw app instance.
|
|
|
|
---
|
|
|
|
## Data Configuration
|
|
|
|
${defaultDatatable
|
|
? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ''}`
|
|
: '**No default datatable configured.** Set \`data.datatable\` in \`raw_app.yaml\` to enable database access.'}
|
|
|
|
### Whitelisted Tables
|
|
|
|
${tables.length > 0
|
|
? `These tables are accessible to this app:\n\n${tables.map(t => `- \`${t}\``).join('\n')}`
|
|
: `**No tables whitelisted.** Add tables to \`data.tables\` in \`raw_app.yaml\`.`}
|
|
|
|
### Adding a Table
|
|
|
|
Edit \`raw_app.yaml\`:
|
|
|
|
\`\`\`yaml
|
|
data:
|
|
datatable: ${defaultDatatable || 'main'}
|
|
${defaultSchema ? `schema: ${defaultSchema}\n ` : ''}tables:
|
|
${tables.length > 0 ? tables.map(t => ` - ${t}`).join('\n') : ' # Add tables here'}
|
|
- ${defaultDatatable || 'main'}/${defaultSchema ? defaultSchema + ':' : ''}new_table # ← Add like this
|
|
\`\`\`
|
|
|
|
**Table reference formats:**
|
|
- \`<datatable>/<table>\` - Table in public schema
|
|
- \`<datatable>/<schema>:<table>\` - Table in specific schema
|
|
|
|
---
|
|
|
|
## Quick Reference
|
|
|
|
**Backend runnable:** Add \`backend/<name>.ts\` (or .py, etc.), then run \`wmill app generate-locks\`
|
|
|
|
**Call from frontend:**
|
|
\`\`\`typescript
|
|
import { backend } from './wmill';
|
|
const result = await backend.<name>({ arg: 'value' });
|
|
\`\`\`
|
|
|
|
**Query datatable (TypeScript):**
|
|
\`\`\`typescript
|
|
const sql = wmill.datatable();
|
|
const rows = await sql\`SELECT * FROM table WHERE id = \${id}\`.fetch();
|
|
\`\`\`
|
|
|
|
**SQL migrations:** Add \`.sql\` files to \`sql_to_apply/\`, run \`wmill app dev\`, then whitelist tables
|
|
|
|
---
|
|
*Run \`wmill app generate-agents\` to refresh. See \`.claude/skills/raw-app\` skill for full documentation.*
|
|
`;
|
|
}
|
|
|
|
/**
|
|
* Generates a simple DATATABLES.md with just the current configuration summary.
|
|
* The detailed schema information is generated by generate_datatables.ts command.
|
|
*/
|
|
export function generateDatatablesDocumentation(data: {
|
|
tables?: string[];
|
|
datatable?: string;
|
|
schema?: string;
|
|
} | undefined): string {
|
|
const tables = data?.tables ?? [];
|
|
const defaultDatatable = data?.datatable;
|
|
const defaultSchema = data?.schema;
|
|
|
|
return `# Data Tables
|
|
|
|
This file contains the database schema information for this app.
|
|
Run \`wmill app generate-agents\` to refresh with current workspace schemas.
|
|
|
|
**For full instructions, see \`AGENTS.md\`.**
|
|
|
|
## Current Configuration
|
|
|
|
${defaultDatatable
|
|
? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ''}`
|
|
: '**No default datatable configured.**'}
|
|
|
|
## Whitelisted Tables
|
|
|
|
${tables.length > 0
|
|
? `${tables.map(t => `- \`${t}\``).join('\n')}`
|
|
: `*No tables whitelisted. Add tables to \`data.tables\` in \`raw_app.yaml\`.*`}
|
|
|
|
---
|
|
|
|
## Schema Information
|
|
|
|
Run \`wmill app generate-agents\` to populate this section with detailed schema information
|
|
from the workspace datatables.
|
|
|
|
---
|
|
*Auto-generated. Run \`wmill app generate-agents\` to refresh schemas.*
|
|
`;
|
|
}
|
|
export function extractInlineScriptsForApps(
|
|
key: string | undefined,
|
|
rec: any,
|
|
pathAssigner: PathAssigner,
|
|
toId: (key: string, val: any) => string,
|
|
removeSchema: boolean,
|
|
): InlineScript[] {
|
|
if (!rec) {
|
|
return [];
|
|
}
|
|
if (typeof rec == "object") {
|
|
return Object.entries(rec).flatMap(([k, v]) => {
|
|
if (k == "inlineScript" && typeof v == "object") {
|
|
rec["type"] = undefined;
|
|
const o: Record<string, any> = v as any;
|
|
const name = toId(key ?? "", rec);
|
|
const [basePathO, ext] = pathAssigner.assignPath(name, o["language"]);
|
|
const basePath = basePathO.replaceAll(SEP, "/");
|
|
const r = [];
|
|
if (o["content"]) {
|
|
const content = o["content"];
|
|
o["content"] = "!inline " + basePath.replaceAll(SEP, "/") + ext;
|
|
r.push({
|
|
path: basePath + ext,
|
|
content: content,
|
|
});
|
|
}
|
|
if (o["lock"] && o["lock"] != "") {
|
|
const lock = o["lock"];
|
|
o["lock"] = "!inline " + basePath.replaceAll(SEP, "/") + "lock";
|
|
r.push({
|
|
path: basePath + "lock",
|
|
content: lock,
|
|
});
|
|
}
|
|
if (removeSchema) {
|
|
o.schema = undefined;
|
|
}
|
|
return r;
|
|
} else {
|
|
return extractInlineScriptsForApps(
|
|
k,
|
|
v,
|
|
pathAssigner,
|
|
toId,
|
|
removeSchema,
|
|
);
|
|
}
|
|
});
|
|
}
|
|
return [];
|
|
}
|
|
|
|
function ZipFSElement(
|
|
zip: JSZip,
|
|
useYaml: boolean,
|
|
defaultTs: "bun" | "deno",
|
|
resourceTypeToFormatExtension: Record<string, string>,
|
|
ignoreCodebaseChanges: boolean,
|
|
): DynFSElement {
|
|
async function _internal_file(
|
|
p: string,
|
|
f: JSZip.JSZipObject,
|
|
): Promise<DynFSElement[]> {
|
|
const kind:
|
|
| "flow"
|
|
| "app"
|
|
| "script"
|
|
| "resource"
|
|
| "other"
|
|
| "raw_app"
|
|
| "dependencies" = isFlowMetadataFile(p)
|
|
? "flow"
|
|
: isAppMetadataFile(p)
|
|
? "app"
|
|
: isRawAppMetadataFile(p)
|
|
? "raw_app"
|
|
: p.endsWith(".script.json")
|
|
? "script"
|
|
: p.endsWith(".resource.json")
|
|
? "resource"
|
|
: p.startsWith("dependencies/")
|
|
? "dependencies"
|
|
: "other";
|
|
|
|
const isJson = p.endsWith(".json");
|
|
|
|
function transformPath() {
|
|
if (kind == "flow") {
|
|
return transformJsonPathToDir(p, "flow");
|
|
} else if (kind == "app") {
|
|
return transformJsonPathToDir(p, "app");
|
|
} else if (kind == "raw_app") {
|
|
return transformJsonPathToDir(p, "raw_app");
|
|
} else if (kind == "dependencies") {
|
|
return p;
|
|
} else {
|
|
return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p;
|
|
}
|
|
}
|
|
|
|
const finalPath = transformPath();
|
|
|
|
const r = [
|
|
{
|
|
isDirectory: kind == "flow" || kind == "app" || kind == "raw_app",
|
|
path: finalPath,
|
|
async *getChildren(): AsyncIterable<DynFSElement> {
|
|
if (kind == "flow") {
|
|
let flow: OpenFlow;
|
|
try {
|
|
flow = JSON.parse(await f.async("text"));
|
|
} catch (error) {
|
|
log.error(`Failed to parse flow.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
let inlineScripts;
|
|
try {
|
|
inlineScripts = extractInlineScriptsForFlows(
|
|
flow.value.modules as any,
|
|
{},
|
|
SEP,
|
|
defaultTs,
|
|
undefined, // pathAssigner - let it create one
|
|
{ skipInlineScriptSuffix: getNonDottedPaths() },
|
|
);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to extract inline scripts for flow at path: ${p}`,
|
|
);
|
|
throw error;
|
|
}
|
|
for (const s of inlineScripts) {
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, s.path),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return s.content;
|
|
},
|
|
};
|
|
}
|
|
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "flow.yaml"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(flow, yamlOptions);
|
|
},
|
|
};
|
|
} else if (kind == "app") {
|
|
let app;
|
|
try {
|
|
app = JSON.parse(await f.async("text"));
|
|
} catch (error) {
|
|
log.error(`Failed to parse app.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
let inlineScripts;
|
|
try {
|
|
inlineScripts = extractInlineScriptsForApps(
|
|
undefined,
|
|
app?.["value"],
|
|
newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }),
|
|
(_, val) => val["name"],
|
|
false,
|
|
);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to extract inline scripts for app at path: ${p}`,
|
|
);
|
|
throw error;
|
|
}
|
|
|
|
for (const s of inlineScripts) {
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, s.path),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return s.content;
|
|
},
|
|
};
|
|
}
|
|
|
|
if (isExecutionModeAnonymous(app)) {
|
|
app.public = true;
|
|
}
|
|
app.policy = undefined;
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "app.yaml"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(app, yamlOptions);
|
|
},
|
|
};
|
|
} else if (kind == "raw_app") {
|
|
let rawApp;
|
|
try {
|
|
rawApp = JSON.parse(await f.async("text"));
|
|
} catch (error) {
|
|
log.error(`Failed to parse app.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
if (rawApp?.["policy"]?.["execution_mode"] == "anonymous") {
|
|
rawApp.public = true;
|
|
}
|
|
// console.log("rawApp", rawApp);
|
|
rawApp.policy = undefined;
|
|
// custom_path is derived from the file path, don't store it
|
|
delete rawApp?.["custom_path"];
|
|
let inlineScripts;
|
|
const value = rawApp?.["value"];
|
|
const runnables = value?.["runnables"] ?? {};
|
|
// console.log("FOOB", value?.["runnables"])
|
|
extractFieldsForRawApps(runnables);
|
|
try {
|
|
inlineScripts = extractInlineScriptsForApps(
|
|
undefined,
|
|
value,
|
|
newRawAppPathAssigner(defaultTs),
|
|
(key, val_) => key,
|
|
true,
|
|
);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to extract inline scripts for raw app at path: ${p}`,
|
|
);
|
|
throw error;
|
|
}
|
|
|
|
try {
|
|
for (const [filePath, content] of Object.entries(
|
|
value?.["files"] ?? [],
|
|
)) {
|
|
// Skip generated/dev-only files and folders
|
|
if (
|
|
filePath.startsWith("/sql_to_apply/") ||
|
|
filePath === "/wmill.d.ts" ||
|
|
filePath === "/AGENTS.md" ||
|
|
filePath === "/DATATABLES.md"
|
|
) {
|
|
continue;
|
|
}
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, filePath.substring(1)),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
if (typeof content !== "string") {
|
|
throw new Error(
|
|
`Content of raw app file ${filePath} is not a string`,
|
|
);
|
|
}
|
|
return content as string;
|
|
},
|
|
};
|
|
}
|
|
} catch (error) {
|
|
log.error(`Failed to extract files for raw app at path: ${p}`);
|
|
throw error;
|
|
}
|
|
|
|
// Yield inline script content and lock files
|
|
for (const s of inlineScripts) {
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, APP_BACKEND_FOLDER, s.path),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return s.content;
|
|
},
|
|
};
|
|
}
|
|
|
|
// Helper to simplify fields for YAML output
|
|
// { type: 'static', value: X } -> { value: X }
|
|
// { type: 'ctx', ctx: X } -> { ctx: X }
|
|
function simplifyFields(fields: Record<string, any> | undefined) {
|
|
if (!fields) return undefined;
|
|
const simplified: Record<string, any> = {};
|
|
for (const [k, v] of Object.entries(fields)) {
|
|
if (typeof v === "object" && v !== null) {
|
|
if (v.type === "static" && v.value !== undefined) {
|
|
simplified[k] = { value: v.value };
|
|
} else if (v.type === "ctx" && v.ctx !== undefined) {
|
|
simplified[k] = { ctx: v.ctx };
|
|
} else {
|
|
// Keep other field types as-is
|
|
simplified[k] = v;
|
|
}
|
|
} else {
|
|
simplified[k] = v;
|
|
}
|
|
}
|
|
return simplified;
|
|
}
|
|
|
|
// Yield each runnable as a separate YAML file in the backend folder
|
|
// For inline scripts, simplify the YAML - inlineScript is not needed since
|
|
// content/lock/language can be derived from sibling files
|
|
for (const [runnableId, runnable] of Object.entries(runnables)) {
|
|
const runnableObj = runnable as Record<string, any>;
|
|
let simplifiedRunnable: Record<string, any>;
|
|
|
|
if (runnableObj.inlineScript) {
|
|
// For inline scripts, remove inlineScript and just keep type: 'inline'
|
|
// plus any other metadata (name, fields, etc.)
|
|
simplifiedRunnable = { type: "inline" };
|
|
|
|
// Copy over any other fields that aren't inlineScript or type
|
|
for (const [key, value] of Object.entries(runnableObj)) {
|
|
if (key !== "inlineScript" && key !== "type") {
|
|
simplifiedRunnable[key] = value;
|
|
}
|
|
}
|
|
} else if (runnableObj.type === "path" && runnableObj.runType) {
|
|
// For path-based runnables, convert from API format to file format
|
|
// { type: "path", runType: "script" } -> { type: "script" }
|
|
// Also remove schema field
|
|
const {
|
|
type: _type,
|
|
runType,
|
|
schema: _schema,
|
|
...rest
|
|
} = runnableObj;
|
|
simplifiedRunnable = {
|
|
type: runType,
|
|
...rest,
|
|
};
|
|
} else {
|
|
// For other runnables, keep as-is
|
|
simplifiedRunnable = runnableObj;
|
|
}
|
|
|
|
// Simplify fields for cleaner YAML output
|
|
if (simplifiedRunnable.fields) {
|
|
simplifiedRunnable.fields = simplifyFields(simplifiedRunnable.fields);
|
|
}
|
|
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(
|
|
finalPath,
|
|
APP_BACKEND_FOLDER,
|
|
`${runnableId}.yaml`,
|
|
),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(simplifiedRunnable, yamlOptions);
|
|
},
|
|
};
|
|
}
|
|
|
|
// Extract data field from value before deleting it
|
|
const data = value?.["data"];
|
|
if (data) {
|
|
rawApp.data = data;
|
|
}
|
|
|
|
// Remove runnables and value from raw_app.yaml - they are now in separate files
|
|
delete rawApp?.["value"];
|
|
// Don't include runnables in raw_app.yaml anymore
|
|
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "raw_app.yaml"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(rawApp, yamlOptions);
|
|
},
|
|
};
|
|
|
|
// Yield DATATABLES.md documentation file for AI agents
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "DATATABLES.md"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return generateDatatablesDocumentation(data);
|
|
},
|
|
};
|
|
}
|
|
},
|
|
|
|
async getContentText(): Promise<string> {
|
|
const content = await f.async("text");
|
|
|
|
if (kind == "script") {
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse script.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
if (
|
|
parsed["lock"] &&
|
|
parsed["lock"] != "" &&
|
|
parsed["codebase"] == undefined
|
|
) {
|
|
parsed["lock"] =
|
|
"!inline " +
|
|
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
|
|
".lock";
|
|
} else if (parsed["lock"] == "") {
|
|
parsed["lock"] = "";
|
|
} else {
|
|
parsed["lock"] = undefined;
|
|
}
|
|
if (ignoreCodebaseChanges && parsed["codebase"]) {
|
|
parsed["codebase"] = undefined;
|
|
}
|
|
return useYaml
|
|
? yamlStringify(parsed, yamlOptions)
|
|
: JSON.stringify(parsed, null, 2);
|
|
}
|
|
|
|
if (kind == "resource") {
|
|
const content = await f.async("text");
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse resource.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
const formatExtension =
|
|
resourceTypeToFormatExtension[parsed["resource_type"]];
|
|
|
|
if (formatExtension) {
|
|
parsed["value"]["content"] =
|
|
"!inline " +
|
|
removeSuffix(p.replaceAll(SEP, "/"), ".resource.json") +
|
|
".resource.file." +
|
|
formatExtension;
|
|
}
|
|
return useYaml
|
|
? yamlStringify(parsed, yamlOptions)
|
|
: JSON.stringify(parsed, null, 2);
|
|
}
|
|
|
|
return useYaml && isJson && kind != "dependencies"
|
|
? (() => {
|
|
try {
|
|
return yamlStringify(JSON.parse(content), yamlOptions);
|
|
} catch (error) {
|
|
log.error(`Failed to parse JSON content at path: ${p}`);
|
|
throw error;
|
|
}
|
|
})()
|
|
: content;
|
|
},
|
|
},
|
|
];
|
|
if (kind == "script") {
|
|
const content = await f.async("text");
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse script lock content at path: ${p}`);
|
|
throw error;
|
|
}
|
|
const lock = parsed["lock"];
|
|
if (lock && lock != "") {
|
|
r.push({
|
|
isDirectory: false,
|
|
path: removeSuffix(finalPath, ".json") + ".lock",
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return lock;
|
|
},
|
|
});
|
|
}
|
|
}
|
|
if (kind == "resource") {
|
|
const content = await f.async("text");
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse resource file content at path: ${p}`);
|
|
throw error;
|
|
}
|
|
const formatExtension =
|
|
resourceTypeToFormatExtension[parsed["resource_type"]];
|
|
|
|
if (formatExtension) {
|
|
const fileContent: string = parsed["value"]["content"];
|
|
if (typeof fileContent === "string") {
|
|
r.push({
|
|
isDirectory: false,
|
|
path:
|
|
removeSuffix(finalPath, ".resource.json") +
|
|
".resource.file." +
|
|
formatExtension,
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return fileContent;
|
|
},
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return r;
|
|
}
|
|
function _internal_folder(p: string, zip: JSZip): DynFSElement {
|
|
return {
|
|
isDirectory: true,
|
|
path: p,
|
|
async *getChildren(): AsyncIterable<DynFSElement> {
|
|
for (const filename in zip.files) {
|
|
const file = zip.files[filename];
|
|
const totalPath = path.join(p, filename);
|
|
if (file.dir) {
|
|
const e = zip.folder(file.name)!;
|
|
yield _internal_folder(totalPath, e);
|
|
} else {
|
|
const fs = await _internal_file(totalPath, file);
|
|
for (const f of fs) {
|
|
yield f;
|
|
}
|
|
}
|
|
}
|
|
},
|
|
// // deno-lint-ignore require-await
|
|
// async getContentBytes(): Promise<Uint8Array> {
|
|
// throw new Error("Cannot get content of folder");
|
|
// },
|
|
// deno-lint-ignore require-await
|
|
async getContentText(): Promise<string> {
|
|
throw new Error("Cannot get content of folder");
|
|
},
|
|
};
|
|
}
|
|
return _internal_folder("." + SEP, zip);
|
|
}
|
|
|
|
export async function* readDirRecursiveWithIgnore(
|
|
ignore: (path: string, isDirectory: boolean) => boolean,
|
|
root: DynFSElement,
|
|
): AsyncGenerator<{
|
|
path: string;
|
|
ignored: boolean;
|
|
isDirectory: boolean;
|
|
// getContentBytes(): Promise<Uint8Array>;
|
|
getContentText(): Promise<string>;
|
|
}> {
|
|
const stack: {
|
|
path: string;
|
|
isDirectory: boolean;
|
|
ignored: boolean;
|
|
c(): AsyncIterable<DynFSElement>;
|
|
// getContentBytes(): Promise<Uint8Array>;
|
|
getContentText(): Promise<string>;
|
|
}[] = [
|
|
{
|
|
path: root.path,
|
|
ignored: ignore(root.path, root.isDirectory),
|
|
isDirectory: root.isDirectory,
|
|
c: root.getChildren,
|
|
// getContentBytes(): Promise<Uint8Array> {
|
|
// throw undefined;
|
|
// },
|
|
getContentText(): Promise<string> {
|
|
throw undefined;
|
|
},
|
|
},
|
|
];
|
|
|
|
while (stack.length > 0) {
|
|
const e = stack.pop()!;
|
|
yield e;
|
|
for await (const e2 of e.c()) {
|
|
if (e2.isDirectory) {
|
|
const dirName = e2.path.split(SEP).pop();
|
|
if (
|
|
dirName == "node_modules" ||
|
|
dirName == ".claude" ||
|
|
dirName?.startsWith(".")
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
stack.push({
|
|
path: e2.path,
|
|
ignored: e.ignored || ignore(e2.path, e2.isDirectory),
|
|
isDirectory: e2.isDirectory,
|
|
// getContentBytes: e2.getContentBytes,
|
|
getContentText: e2.getContentText,
|
|
c: e2.getChildren,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
type Added = { name: "added"; path: string; content: string };
|
|
type Deleted = { name: "deleted"; path: string };
|
|
type Edit = {
|
|
name: "edited";
|
|
path: string;
|
|
before: string;
|
|
after: string;
|
|
codebase?: string;
|
|
};
|
|
|
|
type Change = Added | Deleted | Edit;
|
|
|
|
export async function elementsToMap(
|
|
els: DynFSElement,
|
|
ignore: (path: string, isDirectory: boolean) => boolean,
|
|
json: boolean,
|
|
skips: Skips,
|
|
specificItems?: SpecificItemsConfig,
|
|
branchOverride?: string,
|
|
isRemote?: boolean,
|
|
): Promise<{ [key: string]: string }> {
|
|
const map: { [key: string]: string } = {};
|
|
const processedBasePaths = new Set<string>();
|
|
// Cache git branch at the start to avoid repeated execSync calls per file
|
|
const cachedBranch = branchOverride ?? getCurrentGitBranch() ?? undefined;
|
|
for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
|
|
// console.log("FOO", entry.path, entry.ignored, entry.isDirectory)
|
|
if (entry.isDirectory || entry.ignored) {
|
|
continue;
|
|
}
|
|
const path = entry.path;
|
|
if (
|
|
!isFileResource(path) &&
|
|
!isRawAppFile(path) &&
|
|
!isWorkspaceDependencies(path)
|
|
) {
|
|
if (json && path.endsWith(".yaml")) continue;
|
|
if (!json && path.endsWith(".json")) continue;
|
|
|
|
if (
|
|
![
|
|
"json",
|
|
"yaml",
|
|
"go",
|
|
"sh",
|
|
"ts",
|
|
"py",
|
|
"sql",
|
|
"gql",
|
|
"ps1",
|
|
"php",
|
|
"js",
|
|
"lock",
|
|
"rs",
|
|
"cs",
|
|
"yml",
|
|
"nu",
|
|
"java",
|
|
"rb",
|
|
// for related places search: ADD_NEW_LANG
|
|
].includes(path.split(".").pop() ?? "")
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (isRawAppFile(path)) {
|
|
const suffix = path.split(getFolderSuffix("raw_app") + SEP).pop();
|
|
if (
|
|
suffix?.startsWith("dist/") ||
|
|
suffix == "wmill.d.ts" ||
|
|
suffix == "package-lock.json" ||
|
|
suffix == "DATATABLES.md"
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (skips.skipResources && isFileResource(path)) continue;
|
|
|
|
const ext = json ? ".json" : ".yaml";
|
|
if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue;
|
|
if (
|
|
!skips.includeTriggers &&
|
|
(path.endsWith(".http_trigger" + ext) ||
|
|
path.endsWith(".websocket_trigger" + ext) ||
|
|
path.endsWith(".kafka_trigger" + ext) ||
|
|
path.endsWith(".nats_trigger" + ext) ||
|
|
path.endsWith(".postgres_trigger" + ext) ||
|
|
path.endsWith(".mqtt_trigger" + ext) ||
|
|
path.endsWith(".sqs_trigger" + ext) ||
|
|
path.endsWith(".gcp_trigger" + ext) ||
|
|
path.endsWith(".email_trigger" + ext) ||
|
|
path.endsWith("_native_trigger" + ext))
|
|
) {
|
|
continue;
|
|
}
|
|
if (!skips.includeUsers && path.endsWith(".user" + ext)) continue;
|
|
if (!skips.includeGroups && path.endsWith(".group" + ext)) continue;
|
|
if (!skips.includeSettings && path === "settings" + ext) continue;
|
|
if (!skips.includeKey && path === "encryption_key") continue;
|
|
if (skips.skipResources && path.endsWith(".resource" + ext)) continue;
|
|
if (skips.skipResourceTypes && path.endsWith(".resource-type" + ext)) {
|
|
continue;
|
|
}
|
|
|
|
// Use getTypeStrFromPath for consistent type detection
|
|
try {
|
|
const fileType = getTypeStrFromPath(path);
|
|
if (skips.skipVariables && fileType === "variable") continue;
|
|
if (skips.skipScripts && fileType === "script") continue;
|
|
if (skips.skipFlows && fileType === "flow") continue;
|
|
if (skips.skipApps && fileType === "app") continue;
|
|
if (skips.skipFolders && fileType === "folder") continue;
|
|
if (
|
|
skips.skipWorkspaceDependencies &&
|
|
fileType === "workspace_dependencies"
|
|
)
|
|
continue;
|
|
} catch {
|
|
// If getTypeStrFromPath can't determine the type, continue processing the file
|
|
}
|
|
|
|
// Handle branch-specific files - skip files for other branches
|
|
if (specificItems && isBranchSpecificFile(path)) {
|
|
if (!isCurrentBranchFile(path, cachedBranch)) {
|
|
// Skip branch-specific files for other branches
|
|
continue;
|
|
}
|
|
}
|
|
|
|
const content = await entry.getContentText();
|
|
|
|
if (skips.skipSecrets && path.endsWith(".variable" + ext)) {
|
|
try {
|
|
let o;
|
|
if (json) {
|
|
try {
|
|
o = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse JSON variable content at path: ${path}`);
|
|
throw error;
|
|
}
|
|
} else {
|
|
try {
|
|
o = yamlParseContent(path, content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse YAML variable content at path: ${path}`);
|
|
throw error;
|
|
}
|
|
}
|
|
if (o["is_secret"]) {
|
|
continue;
|
|
}
|
|
} catch (e) {
|
|
log.warn(`Error reading variable ${path} to check for secrets`);
|
|
}
|
|
}
|
|
|
|
// Handle branch-specific path mapping after all filtering
|
|
if (cachedBranch && isCurrentBranchFile(path, cachedBranch)) {
|
|
// This is a branch-specific file for current branch
|
|
const currentBranch = cachedBranch;
|
|
const basePath = fromBranchSpecificPath(path, currentBranch);
|
|
|
|
// Only use branch-specific files if the item type IS configured as branch-specific
|
|
// AND matches the pattern. Otherwise, skip and use base file instead.
|
|
if (!isItemTypeConfigured(basePath, specificItems)) {
|
|
// Type not configured as branch-specific - skip, use base file instead
|
|
continue;
|
|
}
|
|
if (!isSpecificItem(basePath, specificItems)) {
|
|
// Type configured but doesn't match pattern - skip
|
|
continue;
|
|
}
|
|
|
|
// Type configured AND matches - map to base path
|
|
map[basePath] = content;
|
|
processedBasePaths.add(basePath);
|
|
} else if (!isBranchSpecificFile(path)) {
|
|
// This is a regular base file
|
|
if (processedBasePaths.has(path)) {
|
|
// Skip base file, we already processed branch-specific version
|
|
continue;
|
|
}
|
|
// Skip base file if it's configured as branch-specific (expect branch version)
|
|
// Only for LOCAL files - remote workspace only has base paths
|
|
if (!isRemote && isSpecificItem(path, specificItems)) {
|
|
continue;
|
|
}
|
|
map[path] = content;
|
|
}
|
|
// Note: branch-specific files for other branches are already filtered out earlier
|
|
}
|
|
return map;
|
|
}
|
|
|
|
export interface Skips {
|
|
skipVariables?: boolean | undefined;
|
|
skipResources?: boolean | undefined;
|
|
skipResourceTypes?: boolean | undefined;
|
|
skipSecrets?: boolean | undefined;
|
|
skipScripts?: boolean | undefined;
|
|
skipFlows?: boolean | undefined;
|
|
skipApps?: boolean | undefined;
|
|
skipFolders?: boolean | undefined;
|
|
skipWorkspaceDependencies?: boolean | undefined;
|
|
skipScriptsMetadata?: boolean | undefined;
|
|
includeSchedules?: boolean | undefined;
|
|
includeTriggers?: boolean | undefined;
|
|
includeUsers?: boolean | undefined;
|
|
includeGroups?: boolean | undefined;
|
|
includeSettings?: boolean | undefined;
|
|
includeKey?: boolean | undefined;
|
|
}
|
|
|
|
async function compareDynFSElement(
|
|
els1: DynFSElement,
|
|
els2: DynFSElement | undefined,
|
|
ignore: (path: string, isDirectory: boolean) => boolean,
|
|
json: boolean,
|
|
skips: Skips,
|
|
ignoreMetadataDeletion: boolean,
|
|
codebases: SyncCodebase[],
|
|
ignoreCodebaseChanges: boolean,
|
|
specificItems?: SpecificItemsConfig,
|
|
branchOverride?: string,
|
|
isEls1Remote?: boolean,
|
|
): Promise<Change[]> {
|
|
const [m1, m2] = els2
|
|
? await Promise.all([
|
|
elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote),
|
|
elementsToMap(els2, ignore, json, skips, specificItems, branchOverride, !isEls1Remote),
|
|
])
|
|
: [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), {}];
|
|
|
|
const changes: Change[] = [];
|
|
|
|
function parseYaml(k: string, v: string) {
|
|
if (k.endsWith(".script.yaml")) {
|
|
let o: any;
|
|
try {
|
|
o = yamlParseContent(k, v);
|
|
} catch (error) {
|
|
log.error(`Failed to parse script YAML content at path: ${k}`);
|
|
throw error;
|
|
}
|
|
if (typeof o == "object") {
|
|
if (Array.isArray(o?.["lock"])) {
|
|
o["lock"] = o["lock"].join("\n");
|
|
}
|
|
if (o["is_template"] != undefined) {
|
|
delete o["is_template"];
|
|
}
|
|
}
|
|
return o;
|
|
} else {
|
|
try {
|
|
return yamlParseContent(k, v);
|
|
} catch (error) {
|
|
log.error(`Failed to parse YAML content at path: ${k}`);
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
const codebaseChanges: Record<string, string> = {};
|
|
|
|
for (let [k, v] of Object.entries(m1)) {
|
|
const isScriptMetadata =
|
|
k.endsWith(".script.yaml") || k.endsWith(".script.json");
|
|
const skipMetadata = skips.skipScriptsMetadata && isScriptMetadata;
|
|
|
|
if (m2[k] === undefined) {
|
|
if (skipMetadata) {
|
|
continue;
|
|
}
|
|
if (k.startsWith("dependencies/")) {
|
|
log.info(`Adding workspace dependencies file: ${k}`);
|
|
}
|
|
changes.push({ name: "added", path: k, content: v });
|
|
} else {
|
|
if (m2[k] == v) {
|
|
continue;
|
|
} else if (k.endsWith(".json") && !isWorkspaceDependencies(k)) {
|
|
let parsedV, parsedM2;
|
|
try {
|
|
parsedV = JSON.parse(v);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to parse new JSON content for comparison at path: ${k}`,
|
|
);
|
|
throw error;
|
|
}
|
|
try {
|
|
parsedM2 = JSON.parse(m2[k]);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to parse existing JSON content for comparison at path: ${k}`,
|
|
);
|
|
throw error;
|
|
}
|
|
if (deepEqual(parsedV, parsedM2)) {
|
|
continue;
|
|
}
|
|
} else if (k.endsWith(".yaml")) {
|
|
const before = parseYaml(k, m2[k]);
|
|
const after = parseYaml(k, v);
|
|
if (deepEqual(before, after)) {
|
|
continue;
|
|
}
|
|
if (!ignoreCodebaseChanges) {
|
|
if (before?.codebase != undefined) {
|
|
delete before.codebase;
|
|
m2[k] = yamlStringify(before, yamlOptions);
|
|
}
|
|
if (after?.codebase != undefined) {
|
|
if (before.codebase != after.codebase) {
|
|
codebaseChanges[k] = after.codebase;
|
|
}
|
|
delete after.codebase;
|
|
v = yamlStringify(after, yamlOptions);
|
|
}
|
|
}
|
|
if (skipMetadata) {
|
|
continue;
|
|
}
|
|
}
|
|
changes.push({
|
|
name: "edited",
|
|
path: k,
|
|
after: v,
|
|
before: m2[k],
|
|
codebase: codebaseChanges[k],
|
|
});
|
|
}
|
|
}
|
|
|
|
const remoteCodebase: Record<string, string> = {};
|
|
for (const [k] of Object.entries(m2)) {
|
|
if (m1[k] === undefined) {
|
|
if (
|
|
!ignoreMetadataDeletion ||
|
|
(!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json"))
|
|
) {
|
|
changes.push({ name: "deleted", path: k });
|
|
} else if (k?.endsWith(".script.yaml")) {
|
|
const o = parseYaml(k, m2[k]);
|
|
if (o.codebase != undefined) {
|
|
remoteCodebase[k] = o.codebase;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!ignoreCodebaseChanges) {
|
|
for (const [k, v] of Object.entries(remoteCodebase)) {
|
|
const tsFile = k.replace(".script.yaml", ".ts");
|
|
if (
|
|
changes.find(
|
|
(c) =>
|
|
c.path == tsFile && (c.name == "edited" || c.name == "deleted"),
|
|
)
|
|
) {
|
|
continue;
|
|
}
|
|
const c = findCodebase(tsFile, codebases);
|
|
if ((await c?.getDigest()) != v) {
|
|
changes.push({
|
|
name: "edited",
|
|
path: tsFile,
|
|
codebase: v,
|
|
before: m1[tsFile],
|
|
after: m2[tsFile],
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!ignoreCodebaseChanges) {
|
|
for (const change of changes) {
|
|
const codebase = codebaseChanges[change.path];
|
|
if (!codebase) continue;
|
|
|
|
const tsFile = change.path.replace(".script.yaml", ".ts");
|
|
if (change.name == "edited" && change.path == tsFile) {
|
|
change.codebase = codebase;
|
|
}
|
|
}
|
|
}
|
|
|
|
changes.sort((a, b) =>
|
|
getOrderFromPath(a.path) == getOrderFromPath(b.path)
|
|
? a.path.localeCompare(b.path)
|
|
: getOrderFromPath(a.path) - getOrderFromPath(b.path),
|
|
);
|
|
|
|
return changes;
|
|
}
|
|
|
|
function getOrderFromPath(p: string) {
|
|
const typ = getTypeStrFromPath(p);
|
|
// Order by dependencies: items that others depend on should be pushed first
|
|
if (typ == "settings") {
|
|
return 0;
|
|
} else if (typ == "encryption_key") {
|
|
return 1;
|
|
} else if (typ == "user") {
|
|
return 2;
|
|
} else if (typ == "group") {
|
|
return 3;
|
|
} else if (typ == "folder") {
|
|
return 4;
|
|
} else if (typ == "resource-type") {
|
|
return 5;
|
|
} else if (typ == "variable") {
|
|
return 6;
|
|
} else if (typ == "resource") {
|
|
return 7;
|
|
} else if (typ == "workspace_dependencies") {
|
|
return 8;
|
|
} else if (typ == "script") {
|
|
return 9;
|
|
} else if (typ == "flow") {
|
|
return 10;
|
|
} else if (typ == "raw_app") {
|
|
return 11;
|
|
} else if (typ == "app") {
|
|
return 12;
|
|
} else if (typ == "schedule") {
|
|
return 13;
|
|
} else if (
|
|
typ == "http_trigger" ||
|
|
typ == "websocket_trigger" ||
|
|
typ == "kafka_trigger" ||
|
|
typ == "nats_trigger" ||
|
|
typ == "postgres_trigger" ||
|
|
typ == "mqtt_trigger" ||
|
|
typ == "sqs_trigger" ||
|
|
typ == "gcp_trigger" ||
|
|
typ == "email_trigger" ||
|
|
typ == "native_trigger"
|
|
) {
|
|
return 14;
|
|
} else {
|
|
return 15;
|
|
}
|
|
}
|
|
|
|
const isNotWmillFile = (p: string, isDirectory: boolean) => {
|
|
if (p.endsWith(SEP)) {
|
|
return false;
|
|
}
|
|
if (isDirectory) {
|
|
return (
|
|
!p.startsWith("u" + SEP) &&
|
|
!p.startsWith("f" + SEP) &&
|
|
!p.startsWith("g" + SEP) &&
|
|
!p.startsWith("users" + SEP) &&
|
|
!p.startsWith("groups" + SEP) &&
|
|
!p.startsWith("dependencies" + SEP)
|
|
);
|
|
}
|
|
|
|
try {
|
|
const typ = getTypeStrFromPath(p);
|
|
if (
|
|
typ == "resource-type" ||
|
|
typ == "settings" ||
|
|
typ == "encryption_key"
|
|
) {
|
|
return p.includes(SEP);
|
|
} else {
|
|
return (
|
|
!p.startsWith("u" + SEP) &&
|
|
!p.startsWith("f" + SEP) &&
|
|
!p.startsWith("g" + SEP) &&
|
|
!p.startsWith("users" + SEP) &&
|
|
!p.startsWith("groups" + SEP) &&
|
|
!p.startsWith("dependencies" + SEP)
|
|
);
|
|
}
|
|
} catch {
|
|
return true;
|
|
}
|
|
};
|
|
|
|
export const isWhitelisted = (p: string) => {
|
|
return (
|
|
p == "." + SEP ||
|
|
p == "" ||
|
|
p == "u" ||
|
|
p == "f" ||
|
|
p == "g" ||
|
|
p == "users" ||
|
|
p == "groups" ||
|
|
p == "dependencies"
|
|
);
|
|
};
|
|
|
|
export async function ignoreF(wmillconf: {
|
|
includes?: string[];
|
|
excludes?: string[];
|
|
extraIncludes?: string[];
|
|
skipResourceTypes?: boolean;
|
|
skipWorkspaceDependencies?: boolean;
|
|
json?: boolean;
|
|
includeUsers?: boolean;
|
|
includeGroups?: boolean;
|
|
includeSettings?: boolean;
|
|
includeKey?: boolean;
|
|
}): Promise<(p: string, isDirectory: boolean) => boolean> {
|
|
let whitelist: { approve(file: string): boolean } | undefined = undefined;
|
|
|
|
if (
|
|
(Array.isArray(wmillconf?.includes) && wmillconf?.includes?.length > 0) ||
|
|
(Array.isArray(wmillconf?.excludes) && wmillconf?.excludes?.length > 0)
|
|
) {
|
|
whitelist = {
|
|
approve(file: string): boolean {
|
|
return (
|
|
(!wmillconf.includes ||
|
|
wmillconf.includes?.some((i) => minimatch(file, i))) &&
|
|
(!wmillconf?.excludes ||
|
|
wmillconf.excludes!.every((i) => !minimatch(file, i))) &&
|
|
(!wmillconf.extraIncludes ||
|
|
wmillconf.extraIncludes.length === 0 ||
|
|
wmillconf.extraIncludes.some((i) => minimatch(file, i)))
|
|
);
|
|
},
|
|
};
|
|
}
|
|
|
|
try {
|
|
await Deno.stat(".wmillignore");
|
|
throw Error(".wmillignore is not supported anymore, switch to wmill.yaml");
|
|
} catch {
|
|
//expected
|
|
}
|
|
|
|
// new Gitignore.default({ initialRules: ignoreContent.split("\n")}).ignoreContent).compile();
|
|
|
|
return (p: string, isDirectory: boolean) => {
|
|
const ext = wmillconf.json ? ".json" : ".yaml";
|
|
if (!isDirectory && p.endsWith(".resource-type" + ext)) {
|
|
return wmillconf.skipResourceTypes ?? false;
|
|
}
|
|
|
|
// Special files should bypass path-based filtering when their include flags are set
|
|
if (!isDirectory) {
|
|
try {
|
|
const fileType = getTypeStrFromPath(p);
|
|
if (wmillconf.includeUsers && fileType === "user") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (wmillconf.includeGroups && fileType === "group") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (wmillconf.includeSettings && fileType === "settings") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (wmillconf.includeKey && fileType === "encryption_key") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (
|
|
!wmillconf.skipWorkspaceDependencies &&
|
|
fileType === "workspace_dependencies"
|
|
) {
|
|
return false; // Don't ignore workspace dependencies (they are always included unless explicitly skipped)
|
|
}
|
|
} catch {
|
|
// If getTypeStrFromPath can't determine the type, fall through to normal logic
|
|
}
|
|
}
|
|
|
|
return (
|
|
!isWhitelisted(p) &&
|
|
(isNotWmillFile(p, isDirectory) ||
|
|
(!isDirectory && whitelist != undefined && !whitelist.approve(p)))
|
|
);
|
|
};
|
|
}
|
|
|
|
interface ChangeTracker {
|
|
scripts: string[];
|
|
flows: string[];
|
|
apps: string[];
|
|
rawApps: string[];
|
|
}
|
|
|
|
// deno-lint-ignore no-inner-declarations
|
|
async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) {
|
|
const isScript = exts.some((e) => p.endsWith(e));
|
|
if (isScript) {
|
|
if (isFlowPath(p)) {
|
|
const folder = extractFolderPath(p, "flow")!;
|
|
if (!tracker.flows.includes(folder)) {
|
|
tracker.flows.push(folder);
|
|
}
|
|
} else if (isAppPath(p)) {
|
|
const folder = extractFolderPath(p, "app")!;
|
|
if (!tracker.apps.includes(folder)) {
|
|
tracker.apps.push(folder);
|
|
}
|
|
} else if (isRawAppPath(p)) {
|
|
const folder = extractFolderPath(p, "raw_app")!;
|
|
if (!tracker.rawApps.includes(folder)) {
|
|
tracker.rawApps.push(folder);
|
|
}
|
|
} else {
|
|
if (!tracker.scripts.includes(p)) {
|
|
tracker.scripts.push(p);
|
|
}
|
|
}
|
|
} else if (p.endsWith(".script.yaml") || p.endsWith(".script.json")) {
|
|
try {
|
|
const contentPath = await findContentFile(p);
|
|
if (!contentPath) return;
|
|
if (tracker.scripts.includes(contentPath)) return;
|
|
tracker.scripts.push(contentPath);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
|
|
async function buildTracker(changes: Change[]) {
|
|
const tracker: ChangeTracker = {
|
|
scripts: [],
|
|
flows: [],
|
|
apps: [],
|
|
rawApps: [],
|
|
};
|
|
for (const change of changes) {
|
|
if (change.name == "added" || change.name == "edited") {
|
|
await addToChangedIfNotExists(change.path, tracker);
|
|
}
|
|
}
|
|
return tracker;
|
|
}
|
|
|
|
export async function pull(
|
|
opts: GlobalOptions &
|
|
SyncOptions & { repository?: string; promotion?: string; branch?: string },
|
|
) {
|
|
const originalCliOpts = { ...opts };
|
|
opts = await mergeConfigWithConfigFile(opts);
|
|
|
|
// Validate branch configuration early (skipped when --branch is used)
|
|
try {
|
|
await validateBranchConfiguration(opts, opts.branch);
|
|
} catch (error) {
|
|
if (error instanceof Error && error.message.includes("overrides")) {
|
|
log.error(error.message);
|
|
Deno.exit(1);
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
if (opts.stateful) {
|
|
await ensureDir(path.join(Deno.cwd(), ".wmill"));
|
|
}
|
|
|
|
const workspace = await resolveWorkspace(opts, opts.branch);
|
|
await requireLogin(opts);
|
|
|
|
// Resolve effective sync options with branch awareness
|
|
const effectiveOpts = await resolveEffectiveSyncOptions(
|
|
workspace,
|
|
opts,
|
|
opts.promotion,
|
|
opts.branch,
|
|
);
|
|
|
|
// Extract specific items configuration before merging overwrites gitBranches
|
|
const specificItems = getSpecificItemsForCurrentBranch(opts, opts.branch);
|
|
|
|
// Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides)
|
|
opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts);
|
|
|
|
const codebases = await listSyncCodebases(opts);
|
|
|
|
log.info(
|
|
colors.gray(
|
|
"Computing the files to update locally to match remote (taking wmill.yaml into account)",
|
|
),
|
|
);
|
|
|
|
let resourceTypeToFormatExtension: Record<string, string> = {};
|
|
try {
|
|
resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({
|
|
workspace: workspace.workspaceId,
|
|
})) as Record<string, string>;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
const zipFile = await downloadZip(
|
|
workspace,
|
|
opts.plainSecrets,
|
|
opts.skipVariables,
|
|
opts.skipResources,
|
|
opts.skipResourceTypes,
|
|
opts.skipSecrets,
|
|
opts.includeSchedules,
|
|
opts.includeTriggers,
|
|
opts.includeUsers,
|
|
opts.includeGroups,
|
|
opts.includeSettings,
|
|
opts.includeKey,
|
|
opts.skipWorkspaceDependencies,
|
|
opts.defaultTs,
|
|
);
|
|
|
|
const remote = ZipFSElement(
|
|
zipFile!,
|
|
!opts.json,
|
|
opts.defaultTs ?? "bun",
|
|
resourceTypeToFormatExtension,
|
|
true,
|
|
);
|
|
|
|
const local = !opts.stateful
|
|
? await FSFSElement(Deno.cwd(), codebases, true)
|
|
: await FSFSElement(path.join(Deno.cwd(), ".wmill"), [], true);
|
|
|
|
const changes = await compareDynFSElement(
|
|
remote,
|
|
local,
|
|
await ignoreF(opts),
|
|
opts.json ?? false,
|
|
opts,
|
|
false,
|
|
codebases,
|
|
true,
|
|
specificItems,
|
|
opts.branch,
|
|
true, // els1 (remote) is the remote source
|
|
);
|
|
|
|
log.info(
|
|
`remote (${workspace.name}) -> local: ${changes.length} changes to apply`,
|
|
);
|
|
|
|
// Handle JSON output for dry-run
|
|
if (opts.dryRun && opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
return;
|
|
}
|
|
|
|
if (changes.length > 0) {
|
|
if (!opts.jsonOutput) {
|
|
prettyChanges(changes, specificItems, opts.branch);
|
|
}
|
|
if (opts.dryRun) {
|
|
log.info(colors.gray(`Dry run complete.`));
|
|
return;
|
|
}
|
|
if (
|
|
!opts.yes &&
|
|
!(await Confirm.prompt({
|
|
message: `Do you want to apply these ${changes.length} changes to your local files?`,
|
|
default: true,
|
|
}))
|
|
) {
|
|
return;
|
|
}
|
|
|
|
const conflicts = [];
|
|
|
|
log.info(colors.gray(`Applying changes to files ...`));
|
|
for await (const change of changes) {
|
|
// Determine if this file should be written to a branch-specific path
|
|
let targetPath = change.path;
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
const branchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
);
|
|
if (branchSpecificPath) {
|
|
targetPath = branchSpecificPath;
|
|
}
|
|
}
|
|
|
|
const target = path.join(Deno.cwd(), targetPath);
|
|
const stateTarget = path.join(Deno.cwd(), ".wmill", targetPath);
|
|
if (change.name === "edited") {
|
|
if (opts.stateful) {
|
|
try {
|
|
const currentLocal = await Deno.readTextFile(target);
|
|
if (
|
|
currentLocal !== change.before &&
|
|
currentLocal !== change.after
|
|
) {
|
|
log.info(
|
|
colors.red(
|
|
`Conflict detected on ${change.path}\nBoth local and remote have been modified.`,
|
|
),
|
|
);
|
|
if (opts.failConflicts) {
|
|
conflicts.push({
|
|
local: currentLocal,
|
|
change,
|
|
path: change.path,
|
|
});
|
|
continue;
|
|
} else if (opts.yes) {
|
|
log.info(
|
|
colors.red(
|
|
`Override local version with remote since --yes was passed and no --fail-conflicts.`,
|
|
),
|
|
);
|
|
} else {
|
|
showConflict(change.path, currentLocal, change.after);
|
|
if (
|
|
await Confirm.prompt(
|
|
"Preserve local (push to change remote and avoid seeing this again)?",
|
|
)
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
if (exts.some((e) => change.path.endsWith(e))) {
|
|
log.info(
|
|
`Editing script content of ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
} else if (
|
|
change.path.endsWith(".yaml") ||
|
|
change.path.endsWith(".json")
|
|
) {
|
|
log.info(
|
|
`Editing ${getTypeStrFromPath(change.path)} ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
}
|
|
await Deno.writeTextFile(target, change.after);
|
|
|
|
if (opts.stateful) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
await Deno.copyFile(target, stateTarget);
|
|
}
|
|
} else if (change.name === "added") {
|
|
await ensureDir(path.dirname(target));
|
|
if (opts.stateful) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
log.info(
|
|
`Adding ${getTypeStrFromPath(change.path)} ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
}
|
|
await Deno.writeTextFile(target, change.content);
|
|
log.info(
|
|
`Writing ${getTypeStrFromPath(change.path)} ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
if (opts.stateful) {
|
|
await Deno.copyFile(target, stateTarget);
|
|
}
|
|
} else if (change.name === "deleted") {
|
|
try {
|
|
log.info(
|
|
`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`,
|
|
);
|
|
await Deno.remove(target);
|
|
if (opts.stateful) {
|
|
await Deno.remove(stateTarget);
|
|
}
|
|
} catch {
|
|
if (opts.stateful) {
|
|
await Deno.remove(stateTarget);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (opts.failConflicts) {
|
|
if (conflicts.length > 0) {
|
|
console.error(colors.red(`Conflicts were found`));
|
|
log.info("Conflicts:");
|
|
for (const conflict of conflicts) {
|
|
showConflict(conflict.path, conflict.local, conflict.change.after);
|
|
}
|
|
log.info(
|
|
colors.red(`Please resolve these conflicts manually by either:
|
|
- reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted)
|
|
- pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes
|
|
`),
|
|
);
|
|
Deno.exit(1);
|
|
}
|
|
}
|
|
log.info("All local changes pulled, now updating wmill-lock.yaml");
|
|
await readLockfile(); // ensure wmill-lock.yaml exists
|
|
|
|
const tracker: ChangeTracker = await buildTracker(changes);
|
|
const rawWorkspaceDependencies: Record<string, string> =
|
|
await getRawWorkspaceDependencies();
|
|
|
|
for (const change of tracker.scripts) {
|
|
await generateScriptMetadataInternal(
|
|
change,
|
|
workspace,
|
|
opts,
|
|
false,
|
|
true,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
true,
|
|
);
|
|
}
|
|
for (const change of tracker.flows) {
|
|
log.info(`Updating lock for flow ${change}`);
|
|
await generateFlowLockInternal(
|
|
change,
|
|
false,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
false,
|
|
);
|
|
}
|
|
if (tracker.apps.length > 0) {
|
|
log.info(
|
|
colors.gray(
|
|
`Apps ${tracker.apps.join(
|
|
", ",
|
|
)} inline scripts were changed but ignoring metadata regeneration for now`,
|
|
),
|
|
);
|
|
}
|
|
for (const change of tracker.rawApps) {
|
|
log.info(`Updating lock metadata for raw app ${change}`);
|
|
await generateAppLocksInternal(
|
|
change,
|
|
true,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
}
|
|
for (const change of tracker.apps) {
|
|
log.info(`Updating lock metadata for app ${change}`);
|
|
await generateAppLocksInternal(
|
|
change,
|
|
false,
|
|
false,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
}
|
|
if (opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
message: `All ${changes.length} changes applied locally and wmill-lock.yaml updated`,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
} else {
|
|
log.info(
|
|
colors.bold.green.underline(
|
|
`\nDone! All ${changes.length} changes applied locally and wmill-lock.yaml updated.`,
|
|
),
|
|
);
|
|
}
|
|
} else if (opts.jsonOutput) {
|
|
console.log(
|
|
JSON.stringify(
|
|
{ success: true, message: "No changes to apply", total: 0 },
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig, branchOverride?: string) {
|
|
for (const change of changes) {
|
|
let displayPath = change.path;
|
|
let branchNote = "";
|
|
|
|
// Check if this will be written as a branch-specific file
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
const branchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
branchOverride,
|
|
);
|
|
if (branchSpecificPath) {
|
|
displayPath = branchSpecificPath;
|
|
branchNote = " (branch-specific)";
|
|
}
|
|
}
|
|
|
|
if (change.name === "added") {
|
|
log.info(
|
|
colors.green(
|
|
`+ ${getTypeStrFromPath(change.path)} ` +
|
|
displayPath +
|
|
colors.gray(branchNote),
|
|
),
|
|
);
|
|
} else if (change.name === "deleted") {
|
|
log.info(
|
|
colors.red(
|
|
`- ${getTypeStrFromPath(change.path)} ` +
|
|
displayPath +
|
|
colors.gray(branchNote),
|
|
),
|
|
);
|
|
} else if (change.name === "edited") {
|
|
log.info(
|
|
colors.yellow(
|
|
`~ ${getTypeStrFromPath(change.path)} ` +
|
|
displayPath +
|
|
colors.gray(branchNote) +
|
|
(change.codebase ? ` (codebase changed)` : ""),
|
|
),
|
|
);
|
|
if (change.before != change.after) {
|
|
if (change.path.endsWith(".yaml")) {
|
|
try {
|
|
showDiff(
|
|
yamlStringify(
|
|
yamlParseContent(change.path, change.before),
|
|
yamlOptions,
|
|
),
|
|
yamlStringify(
|
|
yamlParseContent(change.path, change.after),
|
|
yamlOptions,
|
|
),
|
|
);
|
|
} catch {
|
|
showDiff(change.before, change.after);
|
|
}
|
|
} else {
|
|
showDiff(change.before, change.after);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// function prettyDiff(diffs: Difference[]) {
|
|
// for (const diff of diffs) {
|
|
// let pathString = "";
|
|
// for (const pathSegment of diff.path) {
|
|
// if (typeof pathSegment === "string") {
|
|
// pathString += ".";
|
|
// pathString += pathSegment;
|
|
// } else {
|
|
// pathString += "[";
|
|
// pathString += pathSegment;
|
|
// pathString += "]";
|
|
// }
|
|
// }
|
|
// if (diff.type === "REMOVE" || diff.type === "CHANGE") {
|
|
// log.info(colors.red("- " + pathString + " = " + diff.oldValue));
|
|
// }
|
|
// if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
|
// log.info(colors.green("+ " + pathString + " = " + diff.value));
|
|
// }
|
|
// }
|
|
// }
|
|
|
|
function removeSuffix(str: string, suffix: string) {
|
|
return str.slice(0, str.length - suffix.length);
|
|
}
|
|
|
|
export async function push(
|
|
opts: GlobalOptions & SyncOptions & { repository?: string; branch?: string },
|
|
) {
|
|
// Save original CLI options before merging with config file
|
|
const originalCliOpts = { ...opts };
|
|
|
|
// Load configuration from wmill.yaml and merge with CLI options
|
|
opts = await mergeConfigWithConfigFile(opts);
|
|
|
|
// Validate branch configuration early (skipped when --branch is used)
|
|
try {
|
|
await validateBranchConfiguration(opts, opts.branch);
|
|
} catch (error) {
|
|
if (error instanceof Error && error.message.includes("overrides")) {
|
|
log.error(error.message);
|
|
Deno.exit(1);
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
const workspace = await resolveWorkspace(opts, opts.branch);
|
|
await requireLogin(opts);
|
|
|
|
// Resolve effective sync options with branch awareness
|
|
const effectiveOpts = await resolveEffectiveSyncOptions(
|
|
workspace,
|
|
opts,
|
|
opts.promotion,
|
|
opts.branch,
|
|
);
|
|
|
|
// Extract specific items configuration BEFORE merging overwrites gitBranches
|
|
const specificItems = getSpecificItemsForCurrentBranch(opts, opts.branch);
|
|
|
|
// Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides)
|
|
opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts);
|
|
|
|
if (opts.lint) {
|
|
log.info("Running lint validation before push...");
|
|
const lintReport = await runLint(opts);
|
|
printReport(lintReport, !!opts.jsonOutput);
|
|
if (!lintReport.success) {
|
|
log.error(colors.red("Push aborted due to lint failures."));
|
|
Deno.exit(1);
|
|
}
|
|
}
|
|
|
|
if (opts.locksRequired) {
|
|
log.info("Checking for missing locks...");
|
|
const lockIssues = await checkMissingLocks(opts);
|
|
if (lockIssues.length > 0) {
|
|
for (const issue of lockIssues) {
|
|
for (const error of issue.errors) {
|
|
log.error(colors.red(` ${issue.path}: ${error}`));
|
|
}
|
|
}
|
|
log.error(
|
|
colors.red(
|
|
`\nPush aborted: ${lockIssues.length} script(s) missing locks.`,
|
|
),
|
|
);
|
|
Deno.exit(1);
|
|
}
|
|
log.info(colors.green("All scripts have valid locks."));
|
|
}
|
|
|
|
const codebases = await listSyncCodebases(opts);
|
|
if (opts.raw) {
|
|
log.info("--raw is now the default, you can remove it as a flag");
|
|
}
|
|
if (opts.stateful) {
|
|
if (!opts.skipPull) {
|
|
log.info(
|
|
colors.gray("You need to be up-to-date before pushing, pulling first."),
|
|
);
|
|
await pull(opts);
|
|
log.info(colors.green("Pull done, now pushing."));
|
|
log.info("\n");
|
|
}
|
|
}
|
|
|
|
log.info(
|
|
colors.gray(
|
|
"Computing the files to update on the remote to match local (taking wmill.yaml includes/excludes into account)",
|
|
),
|
|
);
|
|
let resourceTypeToFormatExtension: Record<string, string> = {};
|
|
try {
|
|
resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({
|
|
workspace: workspace.workspaceId,
|
|
})) as Record<string, string>;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
const remote = ZipFSElement(
|
|
(await downloadZip(
|
|
workspace,
|
|
opts.plainSecrets,
|
|
opts.skipVariables,
|
|
opts.skipResources,
|
|
opts.skipResourceTypes,
|
|
opts.skipSecrets,
|
|
opts.includeSchedules,
|
|
opts.includeTriggers,
|
|
opts.includeUsers,
|
|
opts.includeGroups,
|
|
opts.includeSettings,
|
|
opts.includeKey,
|
|
opts.skipWorkspaceDependencies,
|
|
opts.defaultTs,
|
|
))!,
|
|
!opts.json,
|
|
opts.defaultTs ?? "bun",
|
|
resourceTypeToFormatExtension,
|
|
false,
|
|
);
|
|
|
|
const local = await FSFSElement(path.join(Deno.cwd(), ""), codebases, false);
|
|
const changes = await compareDynFSElement(
|
|
local,
|
|
remote,
|
|
await ignoreF(opts),
|
|
opts.json ?? false,
|
|
opts,
|
|
true,
|
|
codebases,
|
|
false,
|
|
specificItems,
|
|
opts.branch,
|
|
false, // els1 (local) is not the remote source
|
|
);
|
|
|
|
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
|
|
|
|
const tracker: ChangeTracker = await buildTracker(changes);
|
|
|
|
const staleScripts: string[] = [];
|
|
const staleFlows: string[] = [];
|
|
const staleApps: string[] = [];
|
|
|
|
for (const change of tracker.scripts) {
|
|
const stale = await generateScriptMetadataInternal(
|
|
change,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
false,
|
|
);
|
|
if (stale) {
|
|
staleScripts.push(stale);
|
|
}
|
|
}
|
|
|
|
if (staleScripts.length > 0) {
|
|
log.info("");
|
|
log.warn(
|
|
"Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:",
|
|
);
|
|
for (const stale of staleScripts) {
|
|
log.warn(stale);
|
|
}
|
|
|
|
log.info("");
|
|
}
|
|
|
|
for (const change of tracker.flows) {
|
|
const stale = await generateFlowLockInternal(
|
|
change,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
false,
|
|
true,
|
|
);
|
|
if (stale) {
|
|
staleFlows.push(stale);
|
|
}
|
|
}
|
|
|
|
if (staleFlows.length > 0) {
|
|
log.warn(
|
|
"Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:",
|
|
);
|
|
for (const stale of staleFlows) {
|
|
log.warn(stale);
|
|
}
|
|
log.info("");
|
|
}
|
|
|
|
for (const change of tracker.apps) {
|
|
const stale = await generateAppLocksInternal(
|
|
change,
|
|
false,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
if (stale) {
|
|
staleApps.push(stale);
|
|
}
|
|
}
|
|
|
|
for (const change of tracker.rawApps) {
|
|
const stale = await generateAppLocksInternal(
|
|
change,
|
|
true,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
if (stale) {
|
|
staleApps.push(stale);
|
|
}
|
|
}
|
|
|
|
if (staleApps.length > 0) {
|
|
log.warn(
|
|
"Stale apps locks found, you may want to update them using 'wmill app generate-locks' before pushing:",
|
|
);
|
|
for (const stale of staleApps) {
|
|
log.warn(stale);
|
|
}
|
|
log.info("");
|
|
}
|
|
|
|
await fetchRemoteVersion(workspace);
|
|
|
|
log.info(
|
|
`remote (${workspace.name}) <- local: ${changes.length} changes to apply`,
|
|
);
|
|
// Handle JSON output for dry-run
|
|
if (opts.dryRun && opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
return;
|
|
}
|
|
|
|
if (changes.length > 0) {
|
|
if (!opts.jsonOutput) {
|
|
prettyChanges(changes, specificItems, opts.branch);
|
|
}
|
|
if (opts.dryRun) {
|
|
log.info(colors.gray(`Dry run complete.`));
|
|
return;
|
|
}
|
|
if (
|
|
!opts.yes &&
|
|
!(await Confirm.prompt({
|
|
message: `Do you want to apply these ${changes.length} changes to the remote?`,
|
|
default: true,
|
|
}))
|
|
) {
|
|
return;
|
|
}
|
|
|
|
const start = performance.now();
|
|
log.info(colors.gray(`Applying changes to files ...`));
|
|
|
|
let stateful = opts.stateful;
|
|
if (stateful) {
|
|
try {
|
|
await Deno.stat(path.join(Deno.cwd(), ".wmill"));
|
|
} catch {
|
|
stateful = false;
|
|
}
|
|
}
|
|
|
|
// Group changes by base path (before first dot)
|
|
const groupedChanges = new Map<string, typeof changes>();
|
|
for (const change of changes) {
|
|
const basePath = change.path.split(".")[0];
|
|
if (!groupedChanges.has(basePath)) {
|
|
groupedChanges.set(basePath, []);
|
|
}
|
|
groupedChanges.get(basePath)!.push(change);
|
|
}
|
|
|
|
let parallelizationFactor = opts.parallel ?? 1;
|
|
if (parallelizationFactor <= 0) {
|
|
parallelizationFactor = 1;
|
|
}
|
|
const groupedChangesArray = Array.from(groupedChanges.entries());
|
|
log.info(
|
|
`found changes for ${
|
|
groupedChangesArray.length
|
|
} items with a total of ${groupedChangesArray.reduce(
|
|
(acc, [_, changes]) => acc + changes.length,
|
|
0,
|
|
)} files to process`,
|
|
);
|
|
if (parallelizationFactor > 1) {
|
|
log.info(`Parallelizing ${parallelizationFactor} changes at a time`);
|
|
}
|
|
|
|
// Create a pool of workers that processes items as they become available
|
|
const pool = new Set();
|
|
const queue = [...groupedChangesArray];
|
|
// Cache git branch at the start to avoid repeated execSync calls per change
|
|
const cachedBranchForPush = opts.branch || (isGitRepository() ? getCurrentGitBranch() : null);
|
|
|
|
while (queue.length > 0 || pool.size > 0) {
|
|
// Fill the pool until we reach parallelizationFactor
|
|
while (pool.size < parallelizationFactor && queue.length > 0) {
|
|
let [_basePath, changes] = queue.shift()!;
|
|
const promise = (async () => {
|
|
const alreadySynced: string[] = [];
|
|
const isRawApp = isRawAppFile(changes[0].path);
|
|
if (isRawApp) {
|
|
const deleteRawApp = changes.find(
|
|
(change) =>
|
|
change.name === "deleted" &&
|
|
isRawAppFolderMetadataFile(change.path),
|
|
);
|
|
if (deleteRawApp) {
|
|
changes = [deleteRawApp];
|
|
} else {
|
|
changes.splice(1, changes.length - 1);
|
|
}
|
|
}
|
|
|
|
for await (const change of changes) {
|
|
let stateTarget = undefined;
|
|
if (stateful) {
|
|
try {
|
|
stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
|
|
await Deno.stat(stateTarget);
|
|
} catch {
|
|
stateTarget = undefined;
|
|
}
|
|
}
|
|
|
|
if (change.name === "edited") {
|
|
if (
|
|
await handleScriptMetadata(
|
|
change.path,
|
|
workspace,
|
|
alreadySynced,
|
|
opts.message,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
opts,
|
|
)
|
|
) {
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
continue;
|
|
} else if (
|
|
await handleFile(
|
|
change.path,
|
|
workspace,
|
|
alreadySynced,
|
|
opts.message,
|
|
opts,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
)
|
|
) {
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
continue;
|
|
}
|
|
if (stateTarget) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
log.info(
|
|
`Editing ${getTypeStrFromPath(change.path)} ${change.path}`,
|
|
);
|
|
}
|
|
|
|
if (isFileResource(change.path)) {
|
|
const resourceFilePath = await findResourceFile(change.path);
|
|
if (!alreadySynced.includes(resourceFilePath)) {
|
|
alreadySynced.push(resourceFilePath);
|
|
|
|
const newObj = parseFromPath(
|
|
resourceFilePath,
|
|
await Deno.readTextFile(resourceFilePath),
|
|
);
|
|
|
|
// For branch-specific resources, push to the base path on the workspace server
|
|
// This ensures branch-specific files are stored with their base names in the workspace
|
|
let serverPath = resourceFilePath;
|
|
const currentBranch = cachedBranchForPush;
|
|
|
|
if (currentBranch && isBranchSpecificFile(resourceFilePath)) {
|
|
serverPath = fromBranchSpecificPath(
|
|
resourceFilePath,
|
|
currentBranch,
|
|
);
|
|
}
|
|
|
|
await pushResource(
|
|
workspace.workspaceId,
|
|
serverPath,
|
|
undefined,
|
|
newObj,
|
|
resourceFilePath,
|
|
);
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
const oldObj = parseFromPath(change.path, change.before);
|
|
const newObj = parseFromPath(change.path, change.after);
|
|
|
|
// Check if this is a branch-specific item and get the original branch-specific path
|
|
let originalBranchSpecificPath: string | undefined;
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
originalBranchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
);
|
|
}
|
|
|
|
await pushObj(
|
|
workspace.workspaceId,
|
|
change.path,
|
|
oldObj,
|
|
newObj,
|
|
opts.plainSecrets ?? false,
|
|
alreadySynced,
|
|
opts.message,
|
|
originalBranchSpecificPath,
|
|
);
|
|
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
} else if (change.name === "added") {
|
|
if (
|
|
change.path.endsWith(".script.json") ||
|
|
change.path.endsWith(".script.yaml") ||
|
|
change.path.endsWith(".lock") ||
|
|
isFileResource(change.path)
|
|
) {
|
|
continue;
|
|
} else if (
|
|
await handleFile(
|
|
change.path,
|
|
workspace,
|
|
alreadySynced,
|
|
opts.message,
|
|
opts,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
)
|
|
) {
|
|
continue;
|
|
}
|
|
if (stateTarget) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
log.info(
|
|
`Adding ${getTypeStrFromPath(change.path)} ${change.path}`,
|
|
);
|
|
}
|
|
const obj = parseFromPath(change.path, change.content);
|
|
|
|
// Determine the actual local file path for this change
|
|
// For branch-specific items, we read from branch-specific files but push to base server paths
|
|
let localFilePath = change.path;
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
const branchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
);
|
|
if (branchSpecificPath) {
|
|
localFilePath = branchSpecificPath;
|
|
}
|
|
}
|
|
|
|
await pushObj(
|
|
workspace.workspaceId,
|
|
change.path,
|
|
undefined,
|
|
obj,
|
|
opts.plainSecrets ?? false,
|
|
[],
|
|
opts.message,
|
|
localFilePath, // Pass the actual local file path
|
|
);
|
|
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.content);
|
|
}
|
|
} else if (change.name === "deleted") {
|
|
if (change.path.endsWith(".lock")) {
|
|
continue;
|
|
}
|
|
const typ = getTypeStrFromPath(change.path);
|
|
|
|
if (typ == "script") {
|
|
log.info(`Archiving ${typ} ${change.path}`);
|
|
} else {
|
|
log.info(`Deleting ${typ} ${change.path}`);
|
|
}
|
|
const workspaceId = workspace.workspaceId;
|
|
const target = change.path.replaceAll(SEP, "/");
|
|
switch (typ) {
|
|
case "script": {
|
|
await wmill.archiveScriptByPath({
|
|
workspace: workspaceId,
|
|
path: removeExtensionToPath(target),
|
|
});
|
|
break;
|
|
}
|
|
case "folder":
|
|
await wmill.deleteFolder({
|
|
workspace: workspaceId,
|
|
name: change.path.split(SEP)[1],
|
|
});
|
|
break;
|
|
case "resource":
|
|
await wmill.deleteResource({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".resource.json"),
|
|
});
|
|
break;
|
|
case "resource-type":
|
|
await wmill.deleteResourceType({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".resource-type.json"),
|
|
});
|
|
break;
|
|
case "flow":
|
|
await wmill.deleteFlowByPath({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, getDeleteSuffix("flow", "json")),
|
|
});
|
|
break;
|
|
case "app":
|
|
await wmill.deleteApp({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, getDeleteSuffix("app", "json")),
|
|
});
|
|
break;
|
|
case "raw_app":
|
|
if (isRawAppFolderMetadataFile(target)) {
|
|
// Delete the entire raw app
|
|
await wmill.deleteApp({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, getDeleteSuffix("raw_app", "json")),
|
|
});
|
|
} else {
|
|
const rawAppFolder = extractFolderPath(target, "raw_app");
|
|
let folderExists = false;
|
|
if (rawAppFolder) {
|
|
try {
|
|
await Deno.stat(rawAppFolder);
|
|
folderExists = true;
|
|
} catch {
|
|
// folder doesn't exist
|
|
}
|
|
}
|
|
if (folderExists) {
|
|
// For individual file deletions within a raw app,
|
|
// re-push the entire raw app so the backend gets the updated file list
|
|
// (the deleted file won't be included in the push)
|
|
await pushObj(
|
|
workspaceId,
|
|
target,
|
|
undefined,
|
|
undefined,
|
|
opts.plainSecrets ?? false,
|
|
alreadySynced,
|
|
opts.message,
|
|
);
|
|
} else {
|
|
// The entire raw app folder was deleted locally,
|
|
// delete the raw app on the server
|
|
const remotePath = extractResourceName(target, "raw_app");
|
|
if (remotePath) {
|
|
await wmill.deleteApp({
|
|
workspace: workspaceId,
|
|
path: remotePath,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
case "schedule":
|
|
await wmill.deleteSchedule({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".schedule.json"),
|
|
});
|
|
break;
|
|
case "http_trigger":
|
|
await wmill.deleteHttpTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".http_trigger.json"),
|
|
});
|
|
break;
|
|
case "websocket_trigger":
|
|
await wmill.deleteWebsocketTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".websocket_trigger.json"),
|
|
});
|
|
break;
|
|
case "kafka_trigger":
|
|
await wmill.deleteKafkaTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".kafka_trigger.json"),
|
|
});
|
|
break;
|
|
case "nats_trigger":
|
|
await wmill.deleteNatsTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".nats_trigger.json"),
|
|
});
|
|
break;
|
|
case "postgres_trigger":
|
|
await wmill.deletePostgresTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".postgres_trigger.json"),
|
|
});
|
|
break;
|
|
case "mqtt_trigger":
|
|
await wmill.deleteMqttTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".mqtt_trigger.json"),
|
|
});
|
|
break;
|
|
case "sqs_trigger":
|
|
await wmill.deleteSqsTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".sqs_trigger.json"),
|
|
});
|
|
break;
|
|
case "gcp_trigger":
|
|
await wmill.deleteGcpTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".gcp_trigger.json"),
|
|
});
|
|
break;
|
|
case "email_trigger":
|
|
await wmill.deleteEmailTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".email_trigger.json"),
|
|
});
|
|
break;
|
|
case "native_trigger": {
|
|
const triggerInfo = extractNativeTriggerInfo(change.path);
|
|
if (!triggerInfo) {
|
|
throw new Error(
|
|
`Invalid native trigger path: ${change.path}`
|
|
);
|
|
}
|
|
await wmill.deleteNativeTrigger({
|
|
workspace: workspaceId,
|
|
serviceName: triggerInfo.serviceName as NativeServiceName,
|
|
externalId: triggerInfo.externalId,
|
|
});
|
|
break;
|
|
}
|
|
case "variable":
|
|
await wmill.deleteVariable({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".variable.json"),
|
|
});
|
|
break;
|
|
case "user": {
|
|
const users = await wmill.listUsers({
|
|
workspace: workspaceId,
|
|
});
|
|
|
|
const email = removeSuffix(
|
|
removePathPrefix(change.path, "users"),
|
|
".user.json",
|
|
);
|
|
const user = users.find((u) => u.email === email);
|
|
if (!user) {
|
|
throw new Error(`User ${email} not found`);
|
|
}
|
|
await wmill.deleteUser({
|
|
workspace: workspaceId,
|
|
username: user.username,
|
|
});
|
|
break;
|
|
}
|
|
case "group":
|
|
await wmill.deleteGroup({
|
|
workspace: workspaceId,
|
|
name: removeSuffix(
|
|
removePathPrefix(change.path, "groups"),
|
|
".group.json",
|
|
),
|
|
});
|
|
break;
|
|
case "workspace_dependencies":
|
|
const relativePath = removePathPrefix(
|
|
change.path,
|
|
"dependencies",
|
|
);
|
|
|
|
const res = workspaceDependenciesPathToLanguageAndFilename(
|
|
change.path,
|
|
);
|
|
if (!res) {
|
|
throw new Error(
|
|
`Unknown workspace dependencies file format: ${change.path}`,
|
|
);
|
|
}
|
|
const { name, language } = res;
|
|
|
|
await wmill.deleteWorkspaceDependencies({
|
|
workspace: workspaceId,
|
|
language,
|
|
name,
|
|
});
|
|
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
if (stateTarget) {
|
|
try {
|
|
await Deno.remove(stateTarget);
|
|
} catch {
|
|
// state target may not exist already
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})();
|
|
|
|
pool.add(promise);
|
|
// Remove from pool when complete
|
|
promise.then(() => pool.delete(promise));
|
|
}
|
|
|
|
// Wait for at least one task to complete before continuing
|
|
if (pool.size > 0) {
|
|
await Promise.race(pool);
|
|
}
|
|
}
|
|
if (opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
message: `All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name}`,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
duration_ms: Math.round(performance.now() - start),
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
} else {
|
|
log.info(
|
|
colors.bold.green.underline(
|
|
`\nDone! All ${
|
|
changes.length
|
|
} changes pushed to the remote workspace ${
|
|
workspace.workspaceId
|
|
} named ${workspace.name} (${(performance.now() - start).toFixed(
|
|
0,
|
|
)}ms)`,
|
|
),
|
|
);
|
|
}
|
|
} else if (opts.jsonOutput) {
|
|
console.log(
|
|
JSON.stringify(
|
|
{ success: true, message: "No changes to push", total: 0 },
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
const command = new Command()
|
|
.description(
|
|
"sync local with a remote workspaces or the opposite (push or pull)",
|
|
)
|
|
.action(() =>
|
|
log.info("2 actions available, pull and push. Use -h to display help."),
|
|
)
|
|
.command("pull")
|
|
.description("Pull any remote changes and apply them locally.")
|
|
.option("--yes", "Pull without needing confirmation")
|
|
.option(
|
|
"--dry-run",
|
|
"Show changes that would be pulled without actually pushing",
|
|
)
|
|
.option("--plain-secrets", "Pull secrets as plain text")
|
|
.option("--json", "Use JSON instead of YAML")
|
|
.option("--skip-variables", "Skip syncing variables (including secrets)")
|
|
.option("--skip-secrets", "Skip syncing only secrets variables")
|
|
.option("--skip-resources", "Skip syncing resources")
|
|
.option("--skip-resource-types", "Skip syncing resource types")
|
|
.option("--skip-scripts", "Skip syncing scripts")
|
|
.option("--skip-flows", "Skip syncing flows")
|
|
.option("--skip-apps", "Skip syncing apps")
|
|
.option("--skip-folders", "Skip syncing folders")
|
|
.option(
|
|
"--skip-workspace-dependencies",
|
|
"Skip syncing workspace dependencies",
|
|
)
|
|
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
|
|
.option("--include-schedules", "Include syncing schedules")
|
|
.option("--include-triggers", "Include syncing triggers")
|
|
.option("--include-users", "Include syncing users")
|
|
.option("--include-groups", "Include syncing groups")
|
|
.option("--include-settings", "Include syncing workspace settings")
|
|
.option("--include-key", "Include workspace encryption key")
|
|
.option("--skip-branch-validation", "Skip git branch validation and prompts")
|
|
.option("--json-output", "Output results in JSON format")
|
|
.option(
|
|
"-i --includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes",
|
|
)
|
|
.option(
|
|
"-e --excludes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to NOT take into account. Overrides wmill.yaml excludes",
|
|
)
|
|
.option(
|
|
"--extra-includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy",
|
|
)
|
|
.option(
|
|
"--repository <repo:string>",
|
|
"Specify repository path (e.g., u/user/repo) when multiple repositories exist",
|
|
)
|
|
.option(
|
|
"--promotion <branch:string>",
|
|
"Use promotionOverrides from the specified branch instead of regular overrides",
|
|
)
|
|
.option(
|
|
"--branch <branch:string>",
|
|
"Override the current git branch (works even outside a git repository)",
|
|
)
|
|
// deno-lint-ignore no-explicit-any
|
|
.action(pull as any)
|
|
.command("push")
|
|
.description("Push any local changes and apply them remotely.")
|
|
.option("--yes", "Push without needing confirmation")
|
|
.option(
|
|
"--dry-run",
|
|
"Show changes that would be pushed without actually pushing",
|
|
)
|
|
.option("--plain-secrets", "Push secrets as plain text")
|
|
.option("--json", "Use JSON instead of YAML")
|
|
.option("--skip-variables", "Skip syncing variables (including secrets)")
|
|
.option("--skip-secrets", "Skip syncing only secrets variables")
|
|
.option("--skip-resources", "Skip syncing resources")
|
|
.option("--skip-resource-types", "Skip syncing resource types")
|
|
.option("--skip-scripts", "Skip syncing scripts")
|
|
.option("--skip-flows", "Skip syncing flows")
|
|
.option("--skip-apps", "Skip syncing apps")
|
|
.option("--skip-folders", "Skip syncing folders")
|
|
.option(
|
|
"--skip-workspace-dependencies",
|
|
"Skip syncing workspace dependencies",
|
|
)
|
|
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
|
|
.option("--include-schedules", "Include syncing schedules")
|
|
.option("--include-triggers", "Include syncing triggers")
|
|
.option("--include-users", "Include syncing users")
|
|
.option("--include-groups", "Include syncing groups")
|
|
.option("--include-settings", "Include syncing workspace settings")
|
|
.option("--include-key", "Include workspace encryption key")
|
|
.option("--skip-branch-validation", "Skip git branch validation and prompts")
|
|
.option("--json-output", "Output results in JSON format")
|
|
.option(
|
|
"-i --includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)",
|
|
)
|
|
.option(
|
|
"-e --excludes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to NOT take into account.",
|
|
)
|
|
.option(
|
|
"--extra-includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy",
|
|
)
|
|
.option(
|
|
"--message <message:string>",
|
|
"Include a message that will be added to all scripts/flows/apps updated during this push",
|
|
)
|
|
.option("--parallel <number>", "Number of changes to process in parallel")
|
|
.option(
|
|
"--repository <repo:string>",
|
|
"Specify repository path (e.g., u/user/repo) when multiple repositories exist",
|
|
)
|
|
.option(
|
|
"--branch <branch:string>",
|
|
"Override the current git branch (works even outside a git repository)",
|
|
)
|
|
.option("--lint", "Run lint validation before pushing")
|
|
.option(
|
|
"--locks-required",
|
|
"Fail if scripts or flow inline scripts that need locks have no locks",
|
|
)
|
|
// deno-lint-ignore no-explicit-any
|
|
.action(push as any);
|
|
|
|
export default command;
|