mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-21 16:02:28 +00:00
Pyra
5bbb26c72e
* perf(cli): skip relock more accurate Signed-off-by: pyranota <pyra@duck.com> * Update cli/src/utils/metadata.ts Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> * Update cli/src/commands/flow/flow_metadata.ts Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> * fix Signed-off-by: pyranota <pyra@duck.com> * use structuredClone for safety Signed-off-by: pyranota <pyra@duck.com> * chore: update ee-repo-ref to 592848d59ca2304926fb2bd85d000668a7f46a77 This commit updates the EE repository reference after PR #420 was merged in windmill-ee-private. Previous ee-repo-ref: 931813b75b8260faa13ddc07f36a11607b7e3bf6 New ee-repo-ref: 592848d59ca2304926fb2bd85d000668a7f46a77 Automated by sync-ee-ref workflow. * fix ci Signed-off-by: pyranota <pyra@duck.com> * add simple tests Signed-off-by: pyranota <pyra@duck.com> --------- Signed-off-by: pyranota <pyra@duck.com> Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
3074 lines
94 KiB
TypeScript
3074 lines
94 KiB
TypeScript
import { requireLogin } from "../../core/auth.ts";
|
|
import { fetchVersion, resolveWorkspace } from "../../core/context.ts";
|
|
import {
|
|
colors,
|
|
Command,
|
|
Confirm,
|
|
ensureDir,
|
|
JSZip,
|
|
log,
|
|
minimatch,
|
|
path,
|
|
SEP,
|
|
yamlParseContent,
|
|
yamlStringify,
|
|
} from "../../../deps.ts";
|
|
import * as wmill from "../../../gen/services.gen.ts";
|
|
|
|
import {
|
|
getTypeStrFromPath,
|
|
GlobalOptions,
|
|
parseFromPath,
|
|
pushObj,
|
|
showConflict,
|
|
showDiff,
|
|
extractNativeTriggerInfo,
|
|
} from "../../types.ts";
|
|
import { downloadZip } from "./pull.ts";
|
|
import { runLint, printReport } from "../lint/lint.ts";
|
|
|
|
import {
|
|
exts,
|
|
findContentFile,
|
|
findResourceFile,
|
|
handleScriptMetadata,
|
|
removeExtensionToPath,
|
|
} from "../script/script.ts";
|
|
|
|
import { handleFile } from "../script/script.ts";
|
|
import {
|
|
deepEqual,
|
|
fetchRemoteVersion,
|
|
isFileResource,
|
|
isRawAppFile,
|
|
isWorkspaceDependencies,
|
|
} from "../../utils/utils.ts";
|
|
import {
|
|
getEffectiveSettings,
|
|
mergeConfigWithConfigFile,
|
|
SyncOptions,
|
|
validateBranchConfiguration,
|
|
} from "../../core/conf.ts";
|
|
import {
|
|
fromBranchSpecificPath,
|
|
getBranchSpecificPath,
|
|
getSpecificItemsForCurrentBranch,
|
|
isBranchSpecificFile,
|
|
isCurrentBranchFile,
|
|
isItemTypeConfigured,
|
|
isSpecificItem,
|
|
SpecificItemsConfig,
|
|
} from "../../core/specific_items.ts";
|
|
import { getCurrentGitBranch, isGitRepository } from "../../utils/git.ts";
|
|
import { Workspace } from "../workspace/workspace.ts";
|
|
import { removePathPrefix } from "../../types.ts";
|
|
import { listSyncCodebases, SyncCodebase } from "../../utils/codebase.ts";
|
|
import {
|
|
generateScriptMetadataInternal,
|
|
getRawWorkspaceDependencies,
|
|
readLockfile,
|
|
workspaceDependenciesPathToLanguageAndFilename,
|
|
} from "../../utils/metadata.ts";
|
|
import { OpenFlow, NativeServiceName } from "../../../gen/types.gen.ts";
|
|
import { pushResource } from "../resource/resource.ts";
|
|
import {
|
|
newPathAssigner,
|
|
newRawAppPathAssigner,
|
|
PathAssigner,
|
|
} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
|
|
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
|
|
import { generateFlowLockInternal } from "../flow/flow_metadata.ts";
|
|
import { isExecutionModeAnonymous } from "../app/app.ts";
|
|
import {
|
|
APP_BACKEND_FOLDER,
|
|
generateAppLocksInternal,
|
|
} from "../app/app_metadata.ts";
|
|
import {
|
|
isFlowPath,
|
|
isAppPath,
|
|
isRawAppPath,
|
|
extractFolderPath,
|
|
isFlowMetadataFile,
|
|
isAppMetadataFile,
|
|
isRawAppMetadataFile,
|
|
isRawAppFolderMetadataFile,
|
|
getDeleteSuffix,
|
|
transformJsonPathToDir,
|
|
getFolderSuffix,
|
|
getFolderSuffixWithSep,
|
|
getNonDottedPaths,
|
|
} from "../../utils/resource_folders.ts";
|
|
|
|
// Merge CLI options with effective settings, preserving CLI flags as overrides
|
|
function mergeCliWithEffectiveOptions<
|
|
T extends GlobalOptions & SyncOptions & { repository?: string },
|
|
>(cliOpts: T, effectiveOpts: SyncOptions): T {
|
|
// overlay CLI options on top (undefined cliOpts won't override effectiveOpts)
|
|
return Object.assign({}, effectiveOpts, cliOpts) as T;
|
|
}
|
|
|
|
// Resolve effective sync options using branch-based configuration
|
|
async function resolveEffectiveSyncOptions(
|
|
workspace: Workspace,
|
|
localConfig: SyncOptions,
|
|
promotion?: string,
|
|
branchOverride?: string,
|
|
): Promise<SyncOptions> {
|
|
return await getEffectiveSettings(localConfig, promotion, false, false, branchOverride);
|
|
}
|
|
|
|
type DynFSElement = {
|
|
isDirectory: boolean;
|
|
path: string;
|
|
whitelistedExt?: boolean;
|
|
// getContentBytes(): Promise<Uint8Array>;
|
|
getContentText(): Promise<string>;
|
|
getChildren(): AsyncIterable<DynFSElement>;
|
|
};
|
|
|
|
export function findCodebase(
|
|
path: string,
|
|
codebases: SyncCodebase[],
|
|
): SyncCodebase | undefined {
|
|
if (!path.endsWith(".ts")) {
|
|
return;
|
|
}
|
|
for (const c of codebases) {
|
|
let included = false;
|
|
let excluded = false;
|
|
if (c.includes == undefined || c.includes == null) {
|
|
included = true;
|
|
}
|
|
if (typeof c.includes == "string") {
|
|
c.includes = [c.includes];
|
|
}
|
|
for (const r of c.includes ?? []) {
|
|
if (included) {
|
|
break;
|
|
}
|
|
if (minimatch(path, r)) {
|
|
included = true;
|
|
}
|
|
}
|
|
if (typeof c.excludes == "string") {
|
|
c.excludes = [c.excludes];
|
|
}
|
|
for (const r of c.excludes ?? []) {
|
|
if (minimatch(path, r)) {
|
|
excluded = true;
|
|
}
|
|
}
|
|
if (included && !excluded) {
|
|
return c;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function addCodebaseDigestIfRelevant(
|
|
path: string,
|
|
content: string,
|
|
codebases: SyncCodebase[],
|
|
ignoreCodebaseChanges: boolean,
|
|
): Promise<string> {
|
|
const isScript = path.endsWith(".script.yaml");
|
|
if (!isScript) {
|
|
return content;
|
|
}
|
|
let isTs = true;
|
|
const replacedPath = path.replace(".script.yaml", ".ts");
|
|
try {
|
|
await Deno.stat(replacedPath);
|
|
} catch {
|
|
isTs = false;
|
|
}
|
|
if (!isTs) {
|
|
return content;
|
|
}
|
|
if (isTs) {
|
|
const c = findCodebase(replacedPath, codebases);
|
|
if (c) {
|
|
let parsed: any;
|
|
try {
|
|
parsed = yamlParseContent(path, content);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to parse YAML content for codebase digest at path: ${path}`,
|
|
);
|
|
throw error;
|
|
}
|
|
if (parsed && typeof parsed == "object") {
|
|
if (ignoreCodebaseChanges) {
|
|
parsed["codebase"] = undefined;
|
|
} else {
|
|
parsed["codebase"] = await c.getDigest();
|
|
}
|
|
parsed["lock"] = "";
|
|
return yamlStringify(parsed, yamlOptions);
|
|
} else {
|
|
throw Error(
|
|
`Expected local yaml ${path} to be an object, found: ${content} instead`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
return content;
|
|
}
|
|
|
|
export async function FSFSElement(
|
|
p: string,
|
|
codebases: SyncCodebase[],
|
|
ignoreCodebaseChanges: boolean,
|
|
): Promise<DynFSElement> {
|
|
function _internal_element(
|
|
localP: string,
|
|
isDir: boolean,
|
|
codebases: SyncCodebase[],
|
|
): DynFSElement {
|
|
return {
|
|
isDirectory: isDir,
|
|
path: localP.substring(p.length + 1),
|
|
async *getChildren(): AsyncIterable<DynFSElement> {
|
|
if (!isDir) return [];
|
|
try {
|
|
for await (const e of Deno.readDir(localP)) {
|
|
yield _internal_element(
|
|
path.join(localP, e.name),
|
|
e.isDirectory,
|
|
codebases,
|
|
);
|
|
}
|
|
} catch (e) {
|
|
log.warn(`Error reading dir: ${localP}, ${e}`);
|
|
}
|
|
},
|
|
// async getContentBytes(): Promise<Uint8Array> {
|
|
// return await Deno.readFile(localP);
|
|
// },
|
|
async getContentText(): Promise<string> {
|
|
const content = await Deno.readTextFile(localP);
|
|
const itemPath = localP.substring(p.length + 1);
|
|
const r = await addCodebaseDigestIfRelevant(
|
|
itemPath,
|
|
content,
|
|
codebases,
|
|
ignoreCodebaseChanges,
|
|
);
|
|
return r;
|
|
},
|
|
};
|
|
}
|
|
return _internal_element(p, (await Deno.stat(p)).isDirectory, codebases);
|
|
}
|
|
|
|
function prioritizeName(name: string): string {
|
|
if (name == "version") return "aaa";
|
|
if (name == "id") return "aa";
|
|
if (name == "type") return "ab";
|
|
if (name == "summary") return "ad";
|
|
if (name == "name") return "ae";
|
|
if (name == "display_name") return "af";
|
|
if (name == "description") return "ag";
|
|
if (name == "value") return "ah";
|
|
if (name == "content") return "ai";
|
|
if (name == "modules") return "aj";
|
|
if (name == "failure_module") return "ak";
|
|
if (name == "input_transforms") return "al";
|
|
if (name == "lock") return "az";
|
|
if (name == "locks") return "azz";
|
|
|
|
return name;
|
|
}
|
|
|
|
export const yamlOptions = {
|
|
sortKeys: (a: any, b: any) => {
|
|
return prioritizeName(a).localeCompare(prioritizeName(b));
|
|
},
|
|
noCompatMode: true,
|
|
noRefs: true,
|
|
skipInvalid: true,
|
|
};
|
|
|
|
export interface InlineScript {
|
|
path: string;
|
|
content: string;
|
|
}
|
|
|
|
function extractFields(fields: Record<string, any>) {
|
|
Object.entries(fields).forEach(([k, v]) => {
|
|
if (typeof v == "object") {
|
|
if (v.type == "static") {
|
|
fields[k] = { value: v.value };
|
|
} else if (v.type == "javascript") {
|
|
fields[k] = { expr: v.expr, allowUserResources: v.allowUserResources };
|
|
} else if (v.type == "user") {
|
|
fields[k] = undefined;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
export function extractFieldsForRawApps(runnables: Record<string, any>) {
|
|
Object.values(runnables).forEach((v) => {
|
|
if (typeof v == "object") {
|
|
if (v.fields !== undefined) {
|
|
extractFields(v.fields);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Generates AGENTS.md - app-specific configuration for AI agents working with raw apps.
|
|
* References the raw-app skill for complete documentation and includes instance-specific
|
|
* data configuration (datatable, schema, whitelisted tables).
|
|
*/
|
|
export function generateAgentsDocumentation(data: {
|
|
tables?: string[];
|
|
datatable?: string;
|
|
schema?: string;
|
|
} | undefined): string {
|
|
const tables = data?.tables ?? [];
|
|
const defaultDatatable = data?.datatable;
|
|
const defaultSchema = data?.schema;
|
|
|
|
return `# AI Agent Instructions
|
|
|
|
For complete raw app documentation (app structure, backend runnables, datatables, SQL migrations), use the \`raw-app\` skill.
|
|
|
|
This file contains **app-specific configuration** for this raw app instance.
|
|
|
|
---
|
|
|
|
## Data Configuration
|
|
|
|
${defaultDatatable
|
|
? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ''}`
|
|
: '**No default datatable configured.** Set \`data.datatable\` in \`raw_app.yaml\` to enable database access.'}
|
|
|
|
### Whitelisted Tables
|
|
|
|
${tables.length > 0
|
|
? `These tables are accessible to this app:\n\n${tables.map(t => `- \`${t}\``).join('\n')}`
|
|
: `**No tables whitelisted.** Add tables to \`data.tables\` in \`raw_app.yaml\`.`}
|
|
|
|
### Adding a Table
|
|
|
|
Edit \`raw_app.yaml\`:
|
|
|
|
\`\`\`yaml
|
|
data:
|
|
datatable: ${defaultDatatable || 'main'}
|
|
${defaultSchema ? `schema: ${defaultSchema}\n ` : ''}tables:
|
|
${tables.length > 0 ? tables.map(t => ` - ${t}`).join('\n') : ' # Add tables here'}
|
|
- ${defaultDatatable || 'main'}/${defaultSchema ? defaultSchema + ':' : ''}new_table # ← Add like this
|
|
\`\`\`
|
|
|
|
**Table reference formats:**
|
|
- \`<datatable>/<table>\` - Table in public schema
|
|
- \`<datatable>/<schema>:<table>\` - Table in specific schema
|
|
|
|
---
|
|
|
|
## Quick Reference
|
|
|
|
**Backend runnable:** Add \`backend/<name>.ts\` (or .py, etc.), then run \`wmill app generate-locks\`
|
|
|
|
**Call from frontend:**
|
|
\`\`\`typescript
|
|
import { backend } from './wmill';
|
|
const result = await backend.<name>({ arg: 'value' });
|
|
\`\`\`
|
|
|
|
**Query datatable (TypeScript):**
|
|
\`\`\`typescript
|
|
const sql = wmill.datatable();
|
|
const rows = await sql\`SELECT * FROM table WHERE id = \${id}\`.fetch();
|
|
\`\`\`
|
|
|
|
**SQL migrations:** Add \`.sql\` files to \`sql_to_apply/\`, run \`wmill app dev\`, then whitelist tables
|
|
|
|
---
|
|
*Run \`wmill app generate-agents\` to refresh. See \`.claude/skills/raw-app\` skill for full documentation.*
|
|
`;
|
|
}
|
|
|
|
/**
|
|
* Generates a simple DATATABLES.md with just the current configuration summary.
|
|
* The detailed schema information is generated by generate_datatables.ts command.
|
|
*/
|
|
export function generateDatatablesDocumentation(data: {
|
|
tables?: string[];
|
|
datatable?: string;
|
|
schema?: string;
|
|
} | undefined): string {
|
|
const tables = data?.tables ?? [];
|
|
const defaultDatatable = data?.datatable;
|
|
const defaultSchema = data?.schema;
|
|
|
|
return `# Data Tables
|
|
|
|
This file contains the database schema information for this app.
|
|
Run \`wmill app generate-agents\` to refresh with current workspace schemas.
|
|
|
|
**For full instructions, see \`AGENTS.md\`.**
|
|
|
|
## Current Configuration
|
|
|
|
${defaultDatatable
|
|
? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ''}`
|
|
: '**No default datatable configured.**'}
|
|
|
|
## Whitelisted Tables
|
|
|
|
${tables.length > 0
|
|
? `${tables.map(t => `- \`${t}\``).join('\n')}`
|
|
: `*No tables whitelisted. Add tables to \`data.tables\` in \`raw_app.yaml\`.*`}
|
|
|
|
---
|
|
|
|
## Schema Information
|
|
|
|
Run \`wmill app generate-agents\` to populate this section with detailed schema information
|
|
from the workspace datatables.
|
|
|
|
---
|
|
*Auto-generated. Run \`wmill app generate-agents\` to refresh schemas.*
|
|
`;
|
|
}
|
|
export function extractInlineScriptsForApps(
|
|
key: string | undefined,
|
|
rec: any,
|
|
pathAssigner: PathAssigner,
|
|
toId: (key: string, val: any) => string,
|
|
removeSchema: boolean,
|
|
): InlineScript[] {
|
|
if (!rec) {
|
|
return [];
|
|
}
|
|
if (typeof rec == "object") {
|
|
return Object.entries(rec).flatMap(([k, v]) => {
|
|
if (k == "inlineScript" && typeof v == "object") {
|
|
rec["type"] = undefined;
|
|
const o: Record<string, any> = v as any;
|
|
const name = toId(key ?? "", rec);
|
|
const [basePathO, ext] = pathAssigner.assignPath(name, o["language"]);
|
|
const basePath = basePathO.replaceAll(SEP, "/");
|
|
const r = [];
|
|
if (o["content"]) {
|
|
const content = o["content"];
|
|
o["content"] = "!inline " + basePath.replaceAll(SEP, "/") + ext;
|
|
r.push({
|
|
path: basePath + ext,
|
|
content: content,
|
|
});
|
|
}
|
|
if (o["lock"] && o["lock"] != "") {
|
|
const lock = o["lock"];
|
|
o["lock"] = "!inline " + basePath.replaceAll(SEP, "/") + "lock";
|
|
r.push({
|
|
path: basePath + "lock",
|
|
content: lock,
|
|
});
|
|
}
|
|
if (removeSchema) {
|
|
o.schema = undefined;
|
|
}
|
|
return r;
|
|
} else {
|
|
return extractInlineScriptsForApps(
|
|
k,
|
|
v,
|
|
pathAssigner,
|
|
toId,
|
|
removeSchema,
|
|
);
|
|
}
|
|
});
|
|
}
|
|
return [];
|
|
}
|
|
|
|
function ZipFSElement(
|
|
zip: JSZip,
|
|
useYaml: boolean,
|
|
defaultTs: "bun" | "deno",
|
|
resourceTypeToFormatExtension: Record<string, string>,
|
|
ignoreCodebaseChanges: boolean,
|
|
): DynFSElement {
|
|
async function _internal_file(
|
|
p: string,
|
|
f: JSZip.JSZipObject,
|
|
): Promise<DynFSElement[]> {
|
|
const kind:
|
|
| "flow"
|
|
| "app"
|
|
| "script"
|
|
| "resource"
|
|
| "other"
|
|
| "raw_app"
|
|
| "dependencies" = isFlowMetadataFile(p)
|
|
? "flow"
|
|
: isAppMetadataFile(p)
|
|
? "app"
|
|
: isRawAppMetadataFile(p)
|
|
? "raw_app"
|
|
: p.endsWith(".script.json")
|
|
? "script"
|
|
: p.endsWith(".resource.json")
|
|
? "resource"
|
|
: p.startsWith("dependencies/")
|
|
? "dependencies"
|
|
: "other";
|
|
|
|
const isJson = p.endsWith(".json");
|
|
|
|
function transformPath() {
|
|
if (kind == "flow") {
|
|
return transformJsonPathToDir(p, "flow");
|
|
} else if (kind == "app") {
|
|
return transformJsonPathToDir(p, "app");
|
|
} else if (kind == "raw_app") {
|
|
return transformJsonPathToDir(p, "raw_app");
|
|
} else if (kind == "dependencies") {
|
|
return p;
|
|
} else {
|
|
return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p;
|
|
}
|
|
}
|
|
|
|
const finalPath = transformPath();
|
|
|
|
const r = [
|
|
{
|
|
isDirectory: kind == "flow" || kind == "app" || kind == "raw_app",
|
|
path: finalPath,
|
|
async *getChildren(): AsyncIterable<DynFSElement> {
|
|
if (kind == "flow") {
|
|
let flow: OpenFlow;
|
|
try {
|
|
flow = JSON.parse(await f.async("text"));
|
|
} catch (error) {
|
|
log.error(`Failed to parse flow.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
let inlineScripts;
|
|
try {
|
|
inlineScripts = extractInlineScriptsForFlows(
|
|
flow.value.modules as any,
|
|
{},
|
|
SEP,
|
|
defaultTs,
|
|
undefined, // pathAssigner - let it create one
|
|
{ skipInlineScriptSuffix: getNonDottedPaths() },
|
|
);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to extract inline scripts for flow at path: ${p}`,
|
|
);
|
|
throw error;
|
|
}
|
|
for (const s of inlineScripts) {
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, s.path),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return s.content;
|
|
},
|
|
};
|
|
}
|
|
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "flow.yaml"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(flow, yamlOptions);
|
|
},
|
|
};
|
|
} else if (kind == "app") {
|
|
let app;
|
|
try {
|
|
app = JSON.parse(await f.async("text"));
|
|
} catch (error) {
|
|
log.error(`Failed to parse app.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
let inlineScripts;
|
|
try {
|
|
inlineScripts = extractInlineScriptsForApps(
|
|
undefined,
|
|
app?.["value"],
|
|
newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }),
|
|
(_, val) => val["name"],
|
|
false,
|
|
);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to extract inline scripts for app at path: ${p}`,
|
|
);
|
|
throw error;
|
|
}
|
|
|
|
for (const s of inlineScripts) {
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, s.path),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return s.content;
|
|
},
|
|
};
|
|
}
|
|
|
|
if (isExecutionModeAnonymous(app)) {
|
|
app.public = true;
|
|
}
|
|
app.policy = undefined;
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "app.yaml"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(app, yamlOptions);
|
|
},
|
|
};
|
|
} else if (kind == "raw_app") {
|
|
let rawApp;
|
|
try {
|
|
rawApp = JSON.parse(await f.async("text"));
|
|
} catch (error) {
|
|
log.error(`Failed to parse app.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
if (rawApp?.["policy"]?.["execution_mode"] == "anonymous") {
|
|
rawApp.public = true;
|
|
}
|
|
// console.log("rawApp", rawApp);
|
|
rawApp.policy = undefined;
|
|
// custom_path is derived from the file path, don't store it
|
|
delete rawApp?.["custom_path"];
|
|
let inlineScripts;
|
|
const value = rawApp?.["value"];
|
|
const runnables = value?.["runnables"] ?? {};
|
|
// console.log("FOOB", value?.["runnables"])
|
|
extractFieldsForRawApps(runnables);
|
|
try {
|
|
inlineScripts = extractInlineScriptsForApps(
|
|
undefined,
|
|
value,
|
|
newRawAppPathAssigner(defaultTs),
|
|
(key, val_) => key,
|
|
true,
|
|
);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to extract inline scripts for raw app at path: ${p}`,
|
|
);
|
|
throw error;
|
|
}
|
|
|
|
try {
|
|
for (const [filePath, content] of Object.entries(
|
|
value?.["files"] ?? [],
|
|
)) {
|
|
// Skip generated/dev-only files and folders
|
|
if (
|
|
filePath.startsWith("/sql_to_apply/") ||
|
|
filePath === "/wmill.d.ts" ||
|
|
filePath === "/AGENTS.md" ||
|
|
filePath === "/DATATABLES.md"
|
|
) {
|
|
continue;
|
|
}
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, filePath.substring(1)),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
if (typeof content !== "string") {
|
|
throw new Error(
|
|
`Content of raw app file ${filePath} is not a string`,
|
|
);
|
|
}
|
|
return content as string;
|
|
},
|
|
};
|
|
}
|
|
} catch (error) {
|
|
log.error(`Failed to extract files for raw app at path: ${p}`);
|
|
throw error;
|
|
}
|
|
|
|
// Yield inline script content and lock files
|
|
for (const s of inlineScripts) {
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, APP_BACKEND_FOLDER, s.path),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return s.content;
|
|
},
|
|
};
|
|
}
|
|
|
|
// Helper to simplify fields for YAML output
|
|
// { type: 'static', value: X } -> { value: X }
|
|
// { type: 'ctx', ctx: X } -> { ctx: X }
|
|
function simplifyFields(fields: Record<string, any> | undefined) {
|
|
if (!fields) return undefined;
|
|
const simplified: Record<string, any> = {};
|
|
for (const [k, v] of Object.entries(fields)) {
|
|
if (typeof v === "object" && v !== null) {
|
|
if (v.type === "static" && v.value !== undefined) {
|
|
simplified[k] = { value: v.value };
|
|
} else if (v.type === "ctx" && v.ctx !== undefined) {
|
|
simplified[k] = { ctx: v.ctx };
|
|
} else {
|
|
// Keep other field types as-is
|
|
simplified[k] = v;
|
|
}
|
|
} else {
|
|
simplified[k] = v;
|
|
}
|
|
}
|
|
return simplified;
|
|
}
|
|
|
|
// Yield each runnable as a separate YAML file in the backend folder
|
|
// For inline scripts, simplify the YAML - inlineScript is not needed since
|
|
// content/lock/language can be derived from sibling files
|
|
for (const [runnableId, runnable] of Object.entries(runnables)) {
|
|
const runnableObj = runnable as Record<string, any>;
|
|
let simplifiedRunnable: Record<string, any>;
|
|
|
|
if (runnableObj.inlineScript) {
|
|
// For inline scripts, remove inlineScript and just keep type: 'inline'
|
|
// plus any other metadata (name, fields, etc.)
|
|
simplifiedRunnable = { type: "inline" };
|
|
|
|
// Copy over any other fields that aren't inlineScript or type
|
|
for (const [key, value] of Object.entries(runnableObj)) {
|
|
if (key !== "inlineScript" && key !== "type") {
|
|
simplifiedRunnable[key] = value;
|
|
}
|
|
}
|
|
} else if (runnableObj.type === "path" && runnableObj.runType) {
|
|
// For path-based runnables, convert from API format to file format
|
|
// { type: "path", runType: "script" } -> { type: "script" }
|
|
// Also remove schema field
|
|
const {
|
|
type: _type,
|
|
runType,
|
|
schema: _schema,
|
|
...rest
|
|
} = runnableObj;
|
|
simplifiedRunnable = {
|
|
type: runType,
|
|
...rest,
|
|
};
|
|
} else {
|
|
// For other runnables, keep as-is
|
|
simplifiedRunnable = runnableObj;
|
|
}
|
|
|
|
// Simplify fields for cleaner YAML output
|
|
if (simplifiedRunnable.fields) {
|
|
simplifiedRunnable.fields = simplifyFields(simplifiedRunnable.fields);
|
|
}
|
|
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(
|
|
finalPath,
|
|
APP_BACKEND_FOLDER,
|
|
`${runnableId}.yaml`,
|
|
),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(simplifiedRunnable, yamlOptions);
|
|
},
|
|
};
|
|
}
|
|
|
|
// Extract data field from value before deleting it
|
|
const data = value?.["data"];
|
|
if (data) {
|
|
rawApp.data = data;
|
|
}
|
|
|
|
// Remove runnables and value from raw_app.yaml - they are now in separate files
|
|
delete rawApp?.["value"];
|
|
// Don't include runnables in raw_app.yaml anymore
|
|
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "raw_app.yaml"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return yamlStringify(rawApp, yamlOptions);
|
|
},
|
|
};
|
|
|
|
// Yield DATATABLES.md documentation file for AI agents
|
|
yield {
|
|
isDirectory: false,
|
|
path: path.join(finalPath, "DATATABLES.md"),
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return generateDatatablesDocumentation(data);
|
|
},
|
|
};
|
|
}
|
|
},
|
|
|
|
async getContentText(): Promise<string> {
|
|
const content = await f.async("text");
|
|
|
|
if (kind == "script") {
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse script.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
if (
|
|
parsed["lock"] &&
|
|
parsed["lock"] != "" &&
|
|
parsed["codebase"] == undefined
|
|
) {
|
|
parsed["lock"] =
|
|
"!inline " +
|
|
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
|
|
".lock";
|
|
} else if (parsed["lock"] == "") {
|
|
parsed["lock"] = "";
|
|
} else {
|
|
parsed["lock"] = undefined;
|
|
}
|
|
if (ignoreCodebaseChanges && parsed["codebase"]) {
|
|
parsed["codebase"] = undefined;
|
|
}
|
|
return useYaml
|
|
? yamlStringify(parsed, yamlOptions)
|
|
: JSON.stringify(parsed, null, 2);
|
|
}
|
|
|
|
if (kind == "resource") {
|
|
const content = await f.async("text");
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse resource.yaml at path: ${p}`);
|
|
throw error;
|
|
}
|
|
const formatExtension =
|
|
resourceTypeToFormatExtension[parsed["resource_type"]];
|
|
|
|
if (formatExtension) {
|
|
parsed["value"]["content"] =
|
|
"!inline " +
|
|
removeSuffix(p.replaceAll(SEP, "/"), ".resource.json") +
|
|
".resource.file." +
|
|
formatExtension;
|
|
}
|
|
return useYaml
|
|
? yamlStringify(parsed, yamlOptions)
|
|
: JSON.stringify(parsed, null, 2);
|
|
}
|
|
|
|
return useYaml && isJson && kind != "dependencies"
|
|
? (() => {
|
|
try {
|
|
return yamlStringify(JSON.parse(content), yamlOptions);
|
|
} catch (error) {
|
|
log.error(`Failed to parse JSON content at path: ${p}`);
|
|
throw error;
|
|
}
|
|
})()
|
|
: content;
|
|
},
|
|
},
|
|
];
|
|
if (kind == "script") {
|
|
const content = await f.async("text");
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse script lock content at path: ${p}`);
|
|
throw error;
|
|
}
|
|
const lock = parsed["lock"];
|
|
if (lock && lock != "") {
|
|
r.push({
|
|
isDirectory: false,
|
|
path: removeSuffix(finalPath, ".json") + ".lock",
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return lock;
|
|
},
|
|
});
|
|
}
|
|
}
|
|
if (kind == "resource") {
|
|
const content = await f.async("text");
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse resource file content at path: ${p}`);
|
|
throw error;
|
|
}
|
|
const formatExtension =
|
|
resourceTypeToFormatExtension[parsed["resource_type"]];
|
|
|
|
if (formatExtension) {
|
|
const fileContent: string = parsed["value"]["content"];
|
|
if (typeof fileContent === "string") {
|
|
r.push({
|
|
isDirectory: false,
|
|
path:
|
|
removeSuffix(finalPath, ".resource.json") +
|
|
".resource.file." +
|
|
formatExtension,
|
|
async *getChildren() {},
|
|
// deno-lint-ignore require-await
|
|
async getContentText() {
|
|
return fileContent;
|
|
},
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return r;
|
|
}
|
|
function _internal_folder(p: string, zip: JSZip): DynFSElement {
|
|
return {
|
|
isDirectory: true,
|
|
path: p,
|
|
async *getChildren(): AsyncIterable<DynFSElement> {
|
|
for (const filename in zip.files) {
|
|
const file = zip.files[filename];
|
|
const totalPath = path.join(p, filename);
|
|
if (file.dir) {
|
|
const e = zip.folder(file.name)!;
|
|
yield _internal_folder(totalPath, e);
|
|
} else {
|
|
const fs = await _internal_file(totalPath, file);
|
|
for (const f of fs) {
|
|
yield f;
|
|
}
|
|
}
|
|
}
|
|
},
|
|
// // deno-lint-ignore require-await
|
|
// async getContentBytes(): Promise<Uint8Array> {
|
|
// throw new Error("Cannot get content of folder");
|
|
// },
|
|
// deno-lint-ignore require-await
|
|
async getContentText(): Promise<string> {
|
|
throw new Error("Cannot get content of folder");
|
|
},
|
|
};
|
|
}
|
|
return _internal_folder("." + SEP, zip);
|
|
}
|
|
|
|
export async function* readDirRecursiveWithIgnore(
|
|
ignore: (path: string, isDirectory: boolean) => boolean,
|
|
root: DynFSElement,
|
|
): AsyncGenerator<{
|
|
path: string;
|
|
ignored: boolean;
|
|
isDirectory: boolean;
|
|
// getContentBytes(): Promise<Uint8Array>;
|
|
getContentText(): Promise<string>;
|
|
}> {
|
|
const stack: {
|
|
path: string;
|
|
isDirectory: boolean;
|
|
ignored: boolean;
|
|
c(): AsyncIterable<DynFSElement>;
|
|
// getContentBytes(): Promise<Uint8Array>;
|
|
getContentText(): Promise<string>;
|
|
}[] = [
|
|
{
|
|
path: root.path,
|
|
ignored: ignore(root.path, root.isDirectory),
|
|
isDirectory: root.isDirectory,
|
|
c: root.getChildren,
|
|
// getContentBytes(): Promise<Uint8Array> {
|
|
// throw undefined;
|
|
// },
|
|
getContentText(): Promise<string> {
|
|
throw undefined;
|
|
},
|
|
},
|
|
];
|
|
|
|
while (stack.length > 0) {
|
|
const e = stack.pop()!;
|
|
yield e;
|
|
for await (const e2 of e.c()) {
|
|
if (e2.isDirectory) {
|
|
const dirName = e2.path.split(SEP).pop();
|
|
if (
|
|
dirName == "node_modules" ||
|
|
dirName == ".claude" ||
|
|
dirName?.startsWith(".")
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
stack.push({
|
|
path: e2.path,
|
|
ignored: e.ignored || ignore(e2.path, e2.isDirectory),
|
|
isDirectory: e2.isDirectory,
|
|
// getContentBytes: e2.getContentBytes,
|
|
getContentText: e2.getContentText,
|
|
c: e2.getChildren,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
type Added = { name: "added"; path: string; content: string };
|
|
type Deleted = { name: "deleted"; path: string };
|
|
type Edit = {
|
|
name: "edited";
|
|
path: string;
|
|
before: string;
|
|
after: string;
|
|
codebase?: string;
|
|
};
|
|
|
|
type Change = Added | Deleted | Edit;
|
|
|
|
export async function elementsToMap(
|
|
els: DynFSElement,
|
|
ignore: (path: string, isDirectory: boolean) => boolean,
|
|
json: boolean,
|
|
skips: Skips,
|
|
specificItems?: SpecificItemsConfig,
|
|
branchOverride?: string,
|
|
isRemote?: boolean,
|
|
): Promise<{ [key: string]: string }> {
|
|
const map: { [key: string]: string } = {};
|
|
const processedBasePaths = new Set<string>();
|
|
// Cache git branch at the start to avoid repeated execSync calls per file
|
|
const cachedBranch = branchOverride ?? getCurrentGitBranch() ?? undefined;
|
|
for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
|
|
// console.log("FOO", entry.path, entry.ignored, entry.isDirectory)
|
|
if (entry.isDirectory || entry.ignored) {
|
|
continue;
|
|
}
|
|
const path = entry.path;
|
|
if (
|
|
!isFileResource(path) &&
|
|
!isRawAppFile(path) &&
|
|
!isWorkspaceDependencies(path)
|
|
) {
|
|
if (json && path.endsWith(".yaml")) continue;
|
|
if (!json && path.endsWith(".json")) continue;
|
|
|
|
if (
|
|
![
|
|
"json",
|
|
"yaml",
|
|
"go",
|
|
"sh",
|
|
"ts",
|
|
"py",
|
|
"sql",
|
|
"gql",
|
|
"ps1",
|
|
"php",
|
|
"js",
|
|
"lock",
|
|
"rs",
|
|
"cs",
|
|
"yml",
|
|
"nu",
|
|
"java",
|
|
"rb",
|
|
// for related places search: ADD_NEW_LANG
|
|
].includes(path.split(".").pop() ?? "")
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (isRawAppFile(path)) {
|
|
const suffix = path.split(getFolderSuffix("raw_app") + SEP).pop();
|
|
if (
|
|
suffix?.startsWith("dist/") ||
|
|
suffix == "wmill.d.ts" ||
|
|
suffix == "package-lock.json" ||
|
|
suffix == "DATATABLES.md"
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (skips.skipResources && isFileResource(path)) continue;
|
|
|
|
const ext = json ? ".json" : ".yaml";
|
|
if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue;
|
|
if (
|
|
!skips.includeTriggers &&
|
|
(path.endsWith(".http_trigger" + ext) ||
|
|
path.endsWith(".websocket_trigger" + ext) ||
|
|
path.endsWith(".kafka_trigger" + ext) ||
|
|
path.endsWith(".nats_trigger" + ext) ||
|
|
path.endsWith(".postgres_trigger" + ext) ||
|
|
path.endsWith(".mqtt_trigger" + ext) ||
|
|
path.endsWith(".sqs_trigger" + ext) ||
|
|
path.endsWith(".gcp_trigger" + ext) ||
|
|
path.endsWith(".email_trigger" + ext) ||
|
|
path.endsWith("_native_trigger" + ext))
|
|
) {
|
|
continue;
|
|
}
|
|
if (!skips.includeUsers && path.endsWith(".user" + ext)) continue;
|
|
if (!skips.includeGroups && path.endsWith(".group" + ext)) continue;
|
|
if (!skips.includeSettings && path === "settings" + ext) continue;
|
|
if (!skips.includeKey && path === "encryption_key") continue;
|
|
if (skips.skipResources && path.endsWith(".resource" + ext)) continue;
|
|
if (skips.skipResourceTypes && path.endsWith(".resource-type" + ext)) {
|
|
continue;
|
|
}
|
|
|
|
// Use getTypeStrFromPath for consistent type detection
|
|
try {
|
|
const fileType = getTypeStrFromPath(path);
|
|
if (skips.skipVariables && fileType === "variable") continue;
|
|
if (skips.skipScripts && fileType === "script") continue;
|
|
if (skips.skipFlows && fileType === "flow") continue;
|
|
if (skips.skipApps && fileType === "app") continue;
|
|
if (skips.skipFolders && fileType === "folder") continue;
|
|
if (
|
|
skips.skipWorkspaceDependencies &&
|
|
fileType === "workspace_dependencies"
|
|
)
|
|
continue;
|
|
} catch {
|
|
// If getTypeStrFromPath can't determine the type, continue processing the file
|
|
}
|
|
|
|
// Handle branch-specific files - skip files for other branches
|
|
if (specificItems && isBranchSpecificFile(path)) {
|
|
if (!isCurrentBranchFile(path, cachedBranch)) {
|
|
// Skip branch-specific files for other branches
|
|
continue;
|
|
}
|
|
}
|
|
|
|
const content = await entry.getContentText();
|
|
|
|
if (skips.skipSecrets && path.endsWith(".variable" + ext)) {
|
|
try {
|
|
let o;
|
|
if (json) {
|
|
try {
|
|
o = JSON.parse(content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse JSON variable content at path: ${path}`);
|
|
throw error;
|
|
}
|
|
} else {
|
|
try {
|
|
o = yamlParseContent(path, content);
|
|
} catch (error) {
|
|
log.error(`Failed to parse YAML variable content at path: ${path}`);
|
|
throw error;
|
|
}
|
|
}
|
|
if (o["is_secret"]) {
|
|
continue;
|
|
}
|
|
} catch (e) {
|
|
log.warn(`Error reading variable ${path} to check for secrets`);
|
|
}
|
|
}
|
|
|
|
// Handle branch-specific path mapping after all filtering
|
|
if (cachedBranch && isCurrentBranchFile(path, cachedBranch)) {
|
|
// This is a branch-specific file for current branch
|
|
const currentBranch = cachedBranch;
|
|
const basePath = fromBranchSpecificPath(path, currentBranch);
|
|
|
|
// Only use branch-specific files if the item type IS configured as branch-specific
|
|
// AND matches the pattern. Otherwise, skip and use base file instead.
|
|
if (!isItemTypeConfigured(basePath, specificItems)) {
|
|
// Type not configured as branch-specific - skip, use base file instead
|
|
continue;
|
|
}
|
|
if (!isSpecificItem(basePath, specificItems)) {
|
|
// Type configured but doesn't match pattern - skip
|
|
continue;
|
|
}
|
|
|
|
// Type configured AND matches - map to base path
|
|
map[basePath] = content;
|
|
processedBasePaths.add(basePath);
|
|
} else if (!isBranchSpecificFile(path)) {
|
|
// This is a regular base file
|
|
if (processedBasePaths.has(path)) {
|
|
// Skip base file, we already processed branch-specific version
|
|
continue;
|
|
}
|
|
// Skip base file if it's configured as branch-specific (expect branch version)
|
|
// Only for LOCAL files - remote workspace only has base paths
|
|
if (!isRemote && isSpecificItem(path, specificItems)) {
|
|
continue;
|
|
}
|
|
map[path] = content;
|
|
}
|
|
// Note: branch-specific files for other branches are already filtered out earlier
|
|
}
|
|
return map;
|
|
}
|
|
|
|
export interface Skips {
|
|
skipVariables?: boolean | undefined;
|
|
skipResources?: boolean | undefined;
|
|
skipResourceTypes?: boolean | undefined;
|
|
skipSecrets?: boolean | undefined;
|
|
skipScripts?: boolean | undefined;
|
|
skipFlows?: boolean | undefined;
|
|
skipApps?: boolean | undefined;
|
|
skipFolders?: boolean | undefined;
|
|
skipWorkspaceDependencies?: boolean | undefined;
|
|
skipScriptsMetadata?: boolean | undefined;
|
|
includeSchedules?: boolean | undefined;
|
|
includeTriggers?: boolean | undefined;
|
|
includeUsers?: boolean | undefined;
|
|
includeGroups?: boolean | undefined;
|
|
includeSettings?: boolean | undefined;
|
|
includeKey?: boolean | undefined;
|
|
}
|
|
|
|
async function compareDynFSElement(
|
|
els1: DynFSElement,
|
|
els2: DynFSElement | undefined,
|
|
ignore: (path: string, isDirectory: boolean) => boolean,
|
|
json: boolean,
|
|
skips: Skips,
|
|
ignoreMetadataDeletion: boolean,
|
|
codebases: SyncCodebase[],
|
|
ignoreCodebaseChanges: boolean,
|
|
specificItems?: SpecificItemsConfig,
|
|
branchOverride?: string,
|
|
isEls1Remote?: boolean,
|
|
): Promise<Change[]> {
|
|
const [m1, m2] = els2
|
|
? await Promise.all([
|
|
elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote),
|
|
elementsToMap(els2, ignore, json, skips, specificItems, branchOverride, !isEls1Remote),
|
|
])
|
|
: [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), {}];
|
|
|
|
const changes: Change[] = [];
|
|
|
|
function parseYaml(k: string, v: string) {
|
|
if (k.endsWith(".script.yaml")) {
|
|
let o: any;
|
|
try {
|
|
o = yamlParseContent(k, v);
|
|
} catch (error) {
|
|
log.error(`Failed to parse script YAML content at path: ${k}`);
|
|
throw error;
|
|
}
|
|
if (typeof o == "object") {
|
|
if (Array.isArray(o?.["lock"])) {
|
|
o["lock"] = o["lock"].join("\n");
|
|
}
|
|
if (o["is_template"] != undefined) {
|
|
delete o["is_template"];
|
|
}
|
|
}
|
|
return o;
|
|
} else {
|
|
try {
|
|
return yamlParseContent(k, v);
|
|
} catch (error) {
|
|
log.error(`Failed to parse YAML content at path: ${k}`);
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
const codebaseChanges: Record<string, string> = {};
|
|
|
|
for (let [k, v] of Object.entries(m1)) {
|
|
const isScriptMetadata =
|
|
k.endsWith(".script.yaml") || k.endsWith(".script.json");
|
|
const skipMetadata = skips.skipScriptsMetadata && isScriptMetadata;
|
|
|
|
if (m2[k] === undefined) {
|
|
if (skipMetadata) {
|
|
continue;
|
|
}
|
|
if (k.startsWith("dependencies/")) {
|
|
log.info(`Adding workspace dependencies file: ${k}`);
|
|
}
|
|
changes.push({ name: "added", path: k, content: v });
|
|
} else {
|
|
if (m2[k] == v) {
|
|
continue;
|
|
} else if (k.endsWith(".json") && !isWorkspaceDependencies(k)) {
|
|
let parsedV, parsedM2;
|
|
try {
|
|
parsedV = JSON.parse(v);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to parse new JSON content for comparison at path: ${k}`,
|
|
);
|
|
throw error;
|
|
}
|
|
try {
|
|
parsedM2 = JSON.parse(m2[k]);
|
|
} catch (error) {
|
|
log.error(
|
|
`Failed to parse existing JSON content for comparison at path: ${k}`,
|
|
);
|
|
throw error;
|
|
}
|
|
if (deepEqual(parsedV, parsedM2)) {
|
|
continue;
|
|
}
|
|
} else if (k.endsWith(".yaml")) {
|
|
const before = parseYaml(k, m2[k]);
|
|
const after = parseYaml(k, v);
|
|
if (deepEqual(before, after)) {
|
|
continue;
|
|
}
|
|
if (!ignoreCodebaseChanges) {
|
|
if (before?.codebase != undefined) {
|
|
delete before.codebase;
|
|
m2[k] = yamlStringify(before, yamlOptions);
|
|
}
|
|
if (after?.codebase != undefined) {
|
|
if (before.codebase != after.codebase) {
|
|
codebaseChanges[k] = after.codebase;
|
|
}
|
|
delete after.codebase;
|
|
v = yamlStringify(after, yamlOptions);
|
|
}
|
|
}
|
|
if (skipMetadata) {
|
|
continue;
|
|
}
|
|
}
|
|
changes.push({
|
|
name: "edited",
|
|
path: k,
|
|
after: v,
|
|
before: m2[k],
|
|
codebase: codebaseChanges[k],
|
|
});
|
|
}
|
|
}
|
|
|
|
const remoteCodebase: Record<string, string> = {};
|
|
for (const [k] of Object.entries(m2)) {
|
|
if (m1[k] === undefined) {
|
|
if (
|
|
!ignoreMetadataDeletion ||
|
|
(!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json"))
|
|
) {
|
|
changes.push({ name: "deleted", path: k });
|
|
} else if (k?.endsWith(".script.yaml")) {
|
|
const o = parseYaml(k, m2[k]);
|
|
if (o.codebase != undefined) {
|
|
remoteCodebase[k] = o.codebase;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!ignoreCodebaseChanges) {
|
|
for (const [k, v] of Object.entries(remoteCodebase)) {
|
|
const tsFile = k.replace(".script.yaml", ".ts");
|
|
if (
|
|
changes.find(
|
|
(c) =>
|
|
c.path == tsFile && (c.name == "edited" || c.name == "deleted"),
|
|
)
|
|
) {
|
|
continue;
|
|
}
|
|
const c = findCodebase(tsFile, codebases);
|
|
if ((await c?.getDigest()) != v) {
|
|
changes.push({
|
|
name: "edited",
|
|
path: tsFile,
|
|
codebase: v,
|
|
before: m1[tsFile],
|
|
after: m2[tsFile],
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!ignoreCodebaseChanges) {
|
|
for (const change of changes) {
|
|
const codebase = codebaseChanges[change.path];
|
|
if (!codebase) continue;
|
|
|
|
const tsFile = change.path.replace(".script.yaml", ".ts");
|
|
if (change.name == "edited" && change.path == tsFile) {
|
|
change.codebase = codebase;
|
|
}
|
|
}
|
|
}
|
|
|
|
changes.sort((a, b) =>
|
|
getOrderFromPath(a.path) == getOrderFromPath(b.path)
|
|
? a.path.localeCompare(b.path)
|
|
: getOrderFromPath(a.path) - getOrderFromPath(b.path),
|
|
);
|
|
|
|
return changes;
|
|
}
|
|
|
|
function getOrderFromPath(p: string) {
|
|
const typ = getTypeStrFromPath(p);
|
|
// Order by dependencies: items that others depend on should be pushed first
|
|
if (typ == "settings") {
|
|
return 0;
|
|
} else if (typ == "encryption_key") {
|
|
return 1;
|
|
} else if (typ == "user") {
|
|
return 2;
|
|
} else if (typ == "group") {
|
|
return 3;
|
|
} else if (typ == "folder") {
|
|
return 4;
|
|
} else if (typ == "resource-type") {
|
|
return 5;
|
|
} else if (typ == "variable") {
|
|
return 6;
|
|
} else if (typ == "resource") {
|
|
return 7;
|
|
} else if (typ == "workspace_dependencies") {
|
|
return 8;
|
|
} else if (typ == "script") {
|
|
return 9;
|
|
} else if (typ == "flow") {
|
|
return 10;
|
|
} else if (typ == "raw_app") {
|
|
return 11;
|
|
} else if (typ == "app") {
|
|
return 12;
|
|
} else if (typ == "schedule") {
|
|
return 13;
|
|
} else if (
|
|
typ == "http_trigger" ||
|
|
typ == "websocket_trigger" ||
|
|
typ == "kafka_trigger" ||
|
|
typ == "nats_trigger" ||
|
|
typ == "postgres_trigger" ||
|
|
typ == "mqtt_trigger" ||
|
|
typ == "sqs_trigger" ||
|
|
typ == "gcp_trigger" ||
|
|
typ == "email_trigger" ||
|
|
typ == "native_trigger"
|
|
) {
|
|
return 14;
|
|
} else {
|
|
return 15;
|
|
}
|
|
}
|
|
|
|
const isNotWmillFile = (p: string, isDirectory: boolean) => {
|
|
if (p.endsWith(SEP)) {
|
|
return false;
|
|
}
|
|
if (isDirectory) {
|
|
return (
|
|
!p.startsWith("u" + SEP) &&
|
|
!p.startsWith("f" + SEP) &&
|
|
!p.startsWith("g" + SEP) &&
|
|
!p.startsWith("users" + SEP) &&
|
|
!p.startsWith("groups" + SEP) &&
|
|
!p.startsWith("dependencies" + SEP)
|
|
);
|
|
}
|
|
|
|
try {
|
|
const typ = getTypeStrFromPath(p);
|
|
if (
|
|
typ == "resource-type" ||
|
|
typ == "settings" ||
|
|
typ == "encryption_key"
|
|
) {
|
|
return p.includes(SEP);
|
|
} else {
|
|
return (
|
|
!p.startsWith("u" + SEP) &&
|
|
!p.startsWith("f" + SEP) &&
|
|
!p.startsWith("g" + SEP) &&
|
|
!p.startsWith("users" + SEP) &&
|
|
!p.startsWith("groups" + SEP) &&
|
|
!p.startsWith("dependencies" + SEP)
|
|
);
|
|
}
|
|
} catch {
|
|
return true;
|
|
}
|
|
};
|
|
|
|
export const isWhitelisted = (p: string) => {
|
|
return (
|
|
p == "." + SEP ||
|
|
p == "" ||
|
|
p == "u" ||
|
|
p == "f" ||
|
|
p == "g" ||
|
|
p == "users" ||
|
|
p == "groups" ||
|
|
p == "dependencies"
|
|
);
|
|
};
|
|
|
|
export async function ignoreF(wmillconf: {
|
|
includes?: string[];
|
|
excludes?: string[];
|
|
extraIncludes?: string[];
|
|
skipResourceTypes?: boolean;
|
|
skipWorkspaceDependencies?: boolean;
|
|
json?: boolean;
|
|
includeUsers?: boolean;
|
|
includeGroups?: boolean;
|
|
includeSettings?: boolean;
|
|
includeKey?: boolean;
|
|
}): Promise<(p: string, isDirectory: boolean) => boolean> {
|
|
let whitelist: { approve(file: string): boolean } | undefined = undefined;
|
|
|
|
if (
|
|
(Array.isArray(wmillconf?.includes) && wmillconf?.includes?.length > 0) ||
|
|
(Array.isArray(wmillconf?.excludes) && wmillconf?.excludes?.length > 0)
|
|
) {
|
|
whitelist = {
|
|
approve(file: string): boolean {
|
|
return (
|
|
(!wmillconf.includes ||
|
|
wmillconf.includes?.some((i) => minimatch(file, i))) &&
|
|
(!wmillconf?.excludes ||
|
|
wmillconf.excludes!.every((i) => !minimatch(file, i))) &&
|
|
(!wmillconf.extraIncludes ||
|
|
wmillconf.extraIncludes.length === 0 ||
|
|
wmillconf.extraIncludes.some((i) => minimatch(file, i)))
|
|
);
|
|
},
|
|
};
|
|
}
|
|
|
|
try {
|
|
await Deno.stat(".wmillignore");
|
|
throw Error(".wmillignore is not supported anymore, switch to wmill.yaml");
|
|
} catch {
|
|
//expected
|
|
}
|
|
|
|
// new Gitignore.default({ initialRules: ignoreContent.split("\n")}).ignoreContent).compile();
|
|
|
|
return (p: string, isDirectory: boolean) => {
|
|
const ext = wmillconf.json ? ".json" : ".yaml";
|
|
if (!isDirectory && p.endsWith(".resource-type" + ext)) {
|
|
return wmillconf.skipResourceTypes ?? false;
|
|
}
|
|
|
|
// Special files should bypass path-based filtering when their include flags are set
|
|
if (!isDirectory) {
|
|
try {
|
|
const fileType = getTypeStrFromPath(p);
|
|
if (wmillconf.includeUsers && fileType === "user") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (wmillconf.includeGroups && fileType === "group") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (wmillconf.includeSettings && fileType === "settings") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (wmillconf.includeKey && fileType === "encryption_key") {
|
|
return false; // Don't ignore, always include
|
|
}
|
|
if (
|
|
!wmillconf.skipWorkspaceDependencies &&
|
|
fileType === "workspace_dependencies"
|
|
) {
|
|
return false; // Don't ignore workspace dependencies (they are always included unless explicitly skipped)
|
|
}
|
|
} catch {
|
|
// If getTypeStrFromPath can't determine the type, fall through to normal logic
|
|
}
|
|
}
|
|
|
|
return (
|
|
!isWhitelisted(p) &&
|
|
(isNotWmillFile(p, isDirectory) ||
|
|
(!isDirectory && whitelist != undefined && !whitelist.approve(p)))
|
|
);
|
|
};
|
|
}
|
|
|
|
interface ChangeTracker {
|
|
scripts: string[];
|
|
flows: string[];
|
|
apps: string[];
|
|
rawApps: string[];
|
|
}
|
|
|
|
// deno-lint-ignore no-inner-declarations
|
|
async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) {
|
|
const isScript = exts.some((e) => p.endsWith(e));
|
|
if (isScript) {
|
|
if (isFlowPath(p)) {
|
|
const folder = extractFolderPath(p, "flow")!;
|
|
if (!tracker.flows.includes(folder)) {
|
|
tracker.flows.push(folder);
|
|
}
|
|
} else if (isAppPath(p)) {
|
|
const folder = extractFolderPath(p, "app")!;
|
|
if (!tracker.apps.includes(folder)) {
|
|
tracker.apps.push(folder);
|
|
}
|
|
} else if (isRawAppPath(p)) {
|
|
const folder = extractFolderPath(p, "raw_app")!;
|
|
if (!tracker.rawApps.includes(folder)) {
|
|
tracker.rawApps.push(folder);
|
|
}
|
|
} else {
|
|
if (!tracker.scripts.includes(p)) {
|
|
tracker.scripts.push(p);
|
|
}
|
|
}
|
|
} else if (p.endsWith(".script.yaml") || p.endsWith(".script.json")) {
|
|
try {
|
|
const contentPath = await findContentFile(p);
|
|
if (!contentPath) return;
|
|
if (tracker.scripts.includes(contentPath)) return;
|
|
tracker.scripts.push(contentPath);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
|
|
async function buildTracker(changes: Change[]) {
|
|
const tracker: ChangeTracker = {
|
|
scripts: [],
|
|
flows: [],
|
|
apps: [],
|
|
rawApps: [],
|
|
};
|
|
for (const change of changes) {
|
|
if (change.name == "added" || change.name == "edited") {
|
|
await addToChangedIfNotExists(change.path, tracker);
|
|
}
|
|
}
|
|
return tracker;
|
|
}
|
|
|
|
export async function pull(
|
|
opts: GlobalOptions &
|
|
SyncOptions & { repository?: string; promotion?: string; branch?: string },
|
|
) {
|
|
const originalCliOpts = { ...opts };
|
|
opts = await mergeConfigWithConfigFile(opts);
|
|
|
|
// Validate branch configuration early (skipped when --branch is used)
|
|
try {
|
|
await validateBranchConfiguration(opts, opts.branch);
|
|
} catch (error) {
|
|
if (error instanceof Error && error.message.includes("overrides")) {
|
|
log.error(error.message);
|
|
Deno.exit(1);
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
if (opts.stateful) {
|
|
await ensureDir(path.join(Deno.cwd(), ".wmill"));
|
|
}
|
|
|
|
const workspace = await resolveWorkspace(opts, opts.branch);
|
|
await requireLogin(opts);
|
|
|
|
// Resolve effective sync options with branch awareness
|
|
const effectiveOpts = await resolveEffectiveSyncOptions(
|
|
workspace,
|
|
opts,
|
|
opts.promotion,
|
|
opts.branch,
|
|
);
|
|
|
|
// Extract specific items configuration before merging overwrites gitBranches
|
|
const specificItems = getSpecificItemsForCurrentBranch(opts, opts.branch);
|
|
|
|
// Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides)
|
|
opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts);
|
|
|
|
const codebases = await listSyncCodebases(opts);
|
|
|
|
log.info(
|
|
colors.gray(
|
|
"Computing the files to update locally to match remote (taking wmill.yaml into account)",
|
|
),
|
|
);
|
|
|
|
let resourceTypeToFormatExtension: Record<string, string> = {};
|
|
try {
|
|
resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({
|
|
workspace: workspace.workspaceId,
|
|
})) as Record<string, string>;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
const zipFile = await downloadZip(
|
|
workspace,
|
|
opts.plainSecrets,
|
|
opts.skipVariables,
|
|
opts.skipResources,
|
|
opts.skipResourceTypes,
|
|
opts.skipSecrets,
|
|
opts.includeSchedules,
|
|
opts.includeTriggers,
|
|
opts.includeUsers,
|
|
opts.includeGroups,
|
|
opts.includeSettings,
|
|
opts.includeKey,
|
|
opts.skipWorkspaceDependencies,
|
|
opts.defaultTs,
|
|
);
|
|
|
|
const remote = ZipFSElement(
|
|
zipFile!,
|
|
!opts.json,
|
|
opts.defaultTs ?? "bun",
|
|
resourceTypeToFormatExtension,
|
|
true,
|
|
);
|
|
|
|
const local = !opts.stateful
|
|
? await FSFSElement(Deno.cwd(), codebases, true)
|
|
: await FSFSElement(path.join(Deno.cwd(), ".wmill"), [], true);
|
|
|
|
const changes = await compareDynFSElement(
|
|
remote,
|
|
local,
|
|
await ignoreF(opts),
|
|
opts.json ?? false,
|
|
opts,
|
|
false,
|
|
codebases,
|
|
true,
|
|
specificItems,
|
|
opts.branch,
|
|
true, // els1 (remote) is the remote source
|
|
);
|
|
|
|
log.info(
|
|
`remote (${workspace.name}) -> local: ${changes.length} changes to apply`,
|
|
);
|
|
|
|
// Handle JSON output for dry-run
|
|
if (opts.dryRun && opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
return;
|
|
}
|
|
|
|
if (changes.length > 0) {
|
|
if (!opts.jsonOutput) {
|
|
prettyChanges(changes, specificItems, opts.branch);
|
|
}
|
|
if (opts.dryRun) {
|
|
log.info(colors.gray(`Dry run complete.`));
|
|
return;
|
|
}
|
|
if (
|
|
!opts.yes &&
|
|
!(await Confirm.prompt({
|
|
message: `Do you want to apply these ${changes.length} changes to your local files?`,
|
|
default: true,
|
|
}))
|
|
) {
|
|
return;
|
|
}
|
|
|
|
const conflicts = [];
|
|
|
|
log.info(colors.gray(`Applying changes to files ...`));
|
|
for await (const change of changes) {
|
|
// Determine if this file should be written to a branch-specific path
|
|
let targetPath = change.path;
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
const branchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
);
|
|
if (branchSpecificPath) {
|
|
targetPath = branchSpecificPath;
|
|
}
|
|
}
|
|
|
|
const target = path.join(Deno.cwd(), targetPath);
|
|
const stateTarget = path.join(Deno.cwd(), ".wmill", targetPath);
|
|
if (change.name === "edited") {
|
|
if (opts.stateful) {
|
|
try {
|
|
const currentLocal = await Deno.readTextFile(target);
|
|
if (
|
|
currentLocal !== change.before &&
|
|
currentLocal !== change.after
|
|
) {
|
|
log.info(
|
|
colors.red(
|
|
`Conflict detected on ${change.path}\nBoth local and remote have been modified.`,
|
|
),
|
|
);
|
|
if (opts.failConflicts) {
|
|
conflicts.push({
|
|
local: currentLocal,
|
|
change,
|
|
path: change.path,
|
|
});
|
|
continue;
|
|
} else if (opts.yes) {
|
|
log.info(
|
|
colors.red(
|
|
`Override local version with remote since --yes was passed and no --fail-conflicts.`,
|
|
),
|
|
);
|
|
} else {
|
|
showConflict(change.path, currentLocal, change.after);
|
|
if (
|
|
await Confirm.prompt(
|
|
"Preserve local (push to change remote and avoid seeing this again)?",
|
|
)
|
|
) {
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
if (exts.some((e) => change.path.endsWith(e))) {
|
|
log.info(
|
|
`Editing script content of ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
} else if (
|
|
change.path.endsWith(".yaml") ||
|
|
change.path.endsWith(".json")
|
|
) {
|
|
log.info(
|
|
`Editing ${getTypeStrFromPath(change.path)} ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
}
|
|
await Deno.writeTextFile(target, change.after);
|
|
|
|
if (opts.stateful) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
await Deno.copyFile(target, stateTarget);
|
|
}
|
|
} else if (change.name === "added") {
|
|
await ensureDir(path.dirname(target));
|
|
if (opts.stateful) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
log.info(
|
|
`Adding ${getTypeStrFromPath(change.path)} ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
}
|
|
await Deno.writeTextFile(target, change.content);
|
|
log.info(
|
|
`Writing ${getTypeStrFromPath(change.path)} ${targetPath}${
|
|
targetPath !== change.path
|
|
? colors.gray(` (branch-specific override for ${change.path})`)
|
|
: ""
|
|
}`,
|
|
);
|
|
if (opts.stateful) {
|
|
await Deno.copyFile(target, stateTarget);
|
|
}
|
|
} else if (change.name === "deleted") {
|
|
try {
|
|
log.info(
|
|
`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`,
|
|
);
|
|
await Deno.remove(target);
|
|
if (opts.stateful) {
|
|
await Deno.remove(stateTarget);
|
|
}
|
|
} catch {
|
|
if (opts.stateful) {
|
|
await Deno.remove(stateTarget);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (opts.failConflicts) {
|
|
if (conflicts.length > 0) {
|
|
console.error(colors.red(`Conflicts were found`));
|
|
log.info("Conflicts:");
|
|
for (const conflict of conflicts) {
|
|
showConflict(conflict.path, conflict.local, conflict.change.after);
|
|
}
|
|
log.info(
|
|
colors.red(`Please resolve these conflicts manually by either:
|
|
- reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted)
|
|
- pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes
|
|
`),
|
|
);
|
|
Deno.exit(1);
|
|
}
|
|
}
|
|
log.info("All local changes pulled, now updating wmill-lock.yaml");
|
|
await readLockfile(); // ensure wmill-lock.yaml exists
|
|
|
|
const tracker: ChangeTracker = await buildTracker(changes);
|
|
const rawWorkspaceDependencies: Record<string, string> =
|
|
await getRawWorkspaceDependencies();
|
|
|
|
for (const change of tracker.scripts) {
|
|
await generateScriptMetadataInternal(
|
|
change,
|
|
workspace,
|
|
opts,
|
|
false,
|
|
true,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
true,
|
|
);
|
|
}
|
|
for (const change of tracker.flows) {
|
|
log.info(`Updating lock for flow ${change}`);
|
|
await generateFlowLockInternal(
|
|
change,
|
|
false,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
false,
|
|
);
|
|
}
|
|
if (tracker.apps.length > 0) {
|
|
log.info(
|
|
colors.gray(
|
|
`Apps ${tracker.apps.join(
|
|
", ",
|
|
)} inline scripts were changed but ignoring metadata regeneration for now`,
|
|
),
|
|
);
|
|
}
|
|
for (const change of tracker.rawApps) {
|
|
log.info(`Updating lock metadata for raw app ${change}`);
|
|
await generateAppLocksInternal(
|
|
change,
|
|
true,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
}
|
|
for (const change of tracker.apps) {
|
|
log.info(`Updating lock metadata for app ${change}`);
|
|
await generateAppLocksInternal(
|
|
change,
|
|
false,
|
|
false,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
}
|
|
if (opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
message: `All ${changes.length} changes applied locally and wmill-lock.yaml updated`,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
} else {
|
|
log.info(
|
|
colors.bold.green.underline(
|
|
`\nDone! All ${changes.length} changes applied locally and wmill-lock.yaml updated.`,
|
|
),
|
|
);
|
|
}
|
|
} else if (opts.jsonOutput) {
|
|
console.log(
|
|
JSON.stringify(
|
|
{ success: true, message: "No changes to apply", total: 0 },
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig, branchOverride?: string) {
|
|
for (const change of changes) {
|
|
let displayPath = change.path;
|
|
let branchNote = "";
|
|
|
|
// Check if this will be written as a branch-specific file
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
const branchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
branchOverride,
|
|
);
|
|
if (branchSpecificPath) {
|
|
displayPath = branchSpecificPath;
|
|
branchNote = " (branch-specific)";
|
|
}
|
|
}
|
|
|
|
if (change.name === "added") {
|
|
log.info(
|
|
colors.green(
|
|
`+ ${getTypeStrFromPath(change.path)} ` +
|
|
displayPath +
|
|
colors.gray(branchNote),
|
|
),
|
|
);
|
|
} else if (change.name === "deleted") {
|
|
log.info(
|
|
colors.red(
|
|
`- ${getTypeStrFromPath(change.path)} ` +
|
|
displayPath +
|
|
colors.gray(branchNote),
|
|
),
|
|
);
|
|
} else if (change.name === "edited") {
|
|
log.info(
|
|
colors.yellow(
|
|
`~ ${getTypeStrFromPath(change.path)} ` +
|
|
displayPath +
|
|
colors.gray(branchNote) +
|
|
(change.codebase ? ` (codebase changed)` : ""),
|
|
),
|
|
);
|
|
if (change.before != change.after) {
|
|
if (change.path.endsWith(".yaml")) {
|
|
try {
|
|
showDiff(
|
|
yamlStringify(
|
|
yamlParseContent(change.path, change.before),
|
|
yamlOptions,
|
|
),
|
|
yamlStringify(
|
|
yamlParseContent(change.path, change.after),
|
|
yamlOptions,
|
|
),
|
|
);
|
|
} catch {
|
|
showDiff(change.before, change.after);
|
|
}
|
|
} else {
|
|
showDiff(change.before, change.after);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// function prettyDiff(diffs: Difference[]) {
|
|
// for (const diff of diffs) {
|
|
// let pathString = "";
|
|
// for (const pathSegment of diff.path) {
|
|
// if (typeof pathSegment === "string") {
|
|
// pathString += ".";
|
|
// pathString += pathSegment;
|
|
// } else {
|
|
// pathString += "[";
|
|
// pathString += pathSegment;
|
|
// pathString += "]";
|
|
// }
|
|
// }
|
|
// if (diff.type === "REMOVE" || diff.type === "CHANGE") {
|
|
// log.info(colors.red("- " + pathString + " = " + diff.oldValue));
|
|
// }
|
|
// if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
|
// log.info(colors.green("+ " + pathString + " = " + diff.value));
|
|
// }
|
|
// }
|
|
// }
|
|
|
|
function removeSuffix(str: string, suffix: string) {
|
|
return str.slice(0, str.length - suffix.length);
|
|
}
|
|
|
|
export async function push(
|
|
opts: GlobalOptions & SyncOptions & { repository?: string; branch?: string },
|
|
) {
|
|
// Save original CLI options before merging with config file
|
|
const originalCliOpts = { ...opts };
|
|
|
|
// Load configuration from wmill.yaml and merge with CLI options
|
|
opts = await mergeConfigWithConfigFile(opts);
|
|
|
|
// Validate branch configuration early (skipped when --branch is used)
|
|
try {
|
|
await validateBranchConfiguration(opts, opts.branch);
|
|
} catch (error) {
|
|
if (error instanceof Error && error.message.includes("overrides")) {
|
|
log.error(error.message);
|
|
Deno.exit(1);
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
const workspace = await resolveWorkspace(opts, opts.branch);
|
|
await requireLogin(opts);
|
|
|
|
// Resolve effective sync options with branch awareness
|
|
const effectiveOpts = await resolveEffectiveSyncOptions(
|
|
workspace,
|
|
opts,
|
|
opts.promotion,
|
|
opts.branch,
|
|
);
|
|
|
|
// Extract specific items configuration BEFORE merging overwrites gitBranches
|
|
const specificItems = getSpecificItemsForCurrentBranch(opts, opts.branch);
|
|
|
|
// Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides)
|
|
opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts);
|
|
|
|
if (opts.lint) {
|
|
log.info("Running lint validation before push...");
|
|
const lintReport = await runLint(opts);
|
|
printReport(lintReport, !!opts.jsonOutput);
|
|
if (!lintReport.success) {
|
|
log.error(colors.red("Push aborted due to lint failures."));
|
|
Deno.exit(1);
|
|
}
|
|
}
|
|
|
|
const codebases = await listSyncCodebases(opts);
|
|
if (opts.raw) {
|
|
log.info("--raw is now the default, you can remove it as a flag");
|
|
}
|
|
if (opts.stateful) {
|
|
if (!opts.skipPull) {
|
|
log.info(
|
|
colors.gray("You need to be up-to-date before pushing, pulling first."),
|
|
);
|
|
await pull(opts);
|
|
log.info(colors.green("Pull done, now pushing."));
|
|
log.info("\n");
|
|
}
|
|
}
|
|
|
|
log.info(
|
|
colors.gray(
|
|
"Computing the files to update on the remote to match local (taking wmill.yaml includes/excludes into account)",
|
|
),
|
|
);
|
|
let resourceTypeToFormatExtension: Record<string, string> = {};
|
|
try {
|
|
resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({
|
|
workspace: workspace.workspaceId,
|
|
})) as Record<string, string>;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
const remote = ZipFSElement(
|
|
(await downloadZip(
|
|
workspace,
|
|
opts.plainSecrets,
|
|
opts.skipVariables,
|
|
opts.skipResources,
|
|
opts.skipResourceTypes,
|
|
opts.skipSecrets,
|
|
opts.includeSchedules,
|
|
opts.includeTriggers,
|
|
opts.includeUsers,
|
|
opts.includeGroups,
|
|
opts.includeSettings,
|
|
opts.includeKey,
|
|
opts.skipWorkspaceDependencies,
|
|
opts.defaultTs,
|
|
))!,
|
|
!opts.json,
|
|
opts.defaultTs ?? "bun",
|
|
resourceTypeToFormatExtension,
|
|
false,
|
|
);
|
|
|
|
const local = await FSFSElement(path.join(Deno.cwd(), ""), codebases, false);
|
|
const changes = await compareDynFSElement(
|
|
local,
|
|
remote,
|
|
await ignoreF(opts),
|
|
opts.json ?? false,
|
|
opts,
|
|
true,
|
|
codebases,
|
|
false,
|
|
specificItems,
|
|
opts.branch,
|
|
false, // els1 (local) is not the remote source
|
|
);
|
|
|
|
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
|
|
|
|
const tracker: ChangeTracker = await buildTracker(changes);
|
|
|
|
const staleScripts: string[] = [];
|
|
const staleFlows: string[] = [];
|
|
const staleApps: string[] = [];
|
|
|
|
for (const change of tracker.scripts) {
|
|
const stale = await generateScriptMetadataInternal(
|
|
change,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
false,
|
|
);
|
|
if (stale) {
|
|
staleScripts.push(stale);
|
|
}
|
|
}
|
|
|
|
if (staleScripts.length > 0) {
|
|
log.info("");
|
|
log.warn(
|
|
"Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:",
|
|
);
|
|
for (const stale of staleScripts) {
|
|
log.warn(stale);
|
|
}
|
|
|
|
log.info("");
|
|
}
|
|
|
|
for (const change of tracker.flows) {
|
|
const stale = await generateFlowLockInternal(
|
|
change,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
false,
|
|
true,
|
|
);
|
|
if (stale) {
|
|
staleFlows.push(stale);
|
|
}
|
|
}
|
|
|
|
if (staleFlows.length > 0) {
|
|
log.warn(
|
|
"Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:",
|
|
);
|
|
for (const stale of staleFlows) {
|
|
log.warn(stale);
|
|
}
|
|
log.info("");
|
|
}
|
|
|
|
for (const change of tracker.apps) {
|
|
const stale = await generateAppLocksInternal(
|
|
change,
|
|
false,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
if (stale) {
|
|
staleApps.push(stale);
|
|
}
|
|
}
|
|
|
|
for (const change of tracker.rawApps) {
|
|
const stale = await generateAppLocksInternal(
|
|
change,
|
|
true,
|
|
true,
|
|
workspace,
|
|
opts,
|
|
true,
|
|
true,
|
|
);
|
|
if (stale) {
|
|
staleApps.push(stale);
|
|
}
|
|
}
|
|
|
|
if (staleApps.length > 0) {
|
|
log.warn(
|
|
"Stale apps locks found, you may want to update them using 'wmill app generate-locks' before pushing:",
|
|
);
|
|
for (const stale of staleApps) {
|
|
log.warn(stale);
|
|
}
|
|
log.info("");
|
|
}
|
|
|
|
await fetchRemoteVersion(workspace);
|
|
|
|
log.info(
|
|
`remote (${workspace.name}) <- local: ${changes.length} changes to apply`,
|
|
);
|
|
// Handle JSON output for dry-run
|
|
if (opts.dryRun && opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
return;
|
|
}
|
|
|
|
if (changes.length > 0) {
|
|
if (!opts.jsonOutput) {
|
|
prettyChanges(changes, specificItems, opts.branch);
|
|
}
|
|
if (opts.dryRun) {
|
|
log.info(colors.gray(`Dry run complete.`));
|
|
return;
|
|
}
|
|
if (
|
|
!opts.yes &&
|
|
!(await Confirm.prompt({
|
|
message: `Do you want to apply these ${changes.length} changes to the remote?`,
|
|
default: true,
|
|
}))
|
|
) {
|
|
return;
|
|
}
|
|
|
|
const start = performance.now();
|
|
log.info(colors.gray(`Applying changes to files ...`));
|
|
|
|
let stateful = opts.stateful;
|
|
if (stateful) {
|
|
try {
|
|
await Deno.stat(path.join(Deno.cwd(), ".wmill"));
|
|
} catch {
|
|
stateful = false;
|
|
}
|
|
}
|
|
|
|
// Group changes by base path (before first dot)
|
|
const groupedChanges = new Map<string, typeof changes>();
|
|
for (const change of changes) {
|
|
const basePath = change.path.split(".")[0];
|
|
if (!groupedChanges.has(basePath)) {
|
|
groupedChanges.set(basePath, []);
|
|
}
|
|
groupedChanges.get(basePath)!.push(change);
|
|
}
|
|
|
|
let parallelizationFactor = opts.parallel ?? 1;
|
|
if (parallelizationFactor <= 0) {
|
|
parallelizationFactor = 1;
|
|
}
|
|
const groupedChangesArray = Array.from(groupedChanges.entries());
|
|
log.info(
|
|
`found changes for ${
|
|
groupedChangesArray.length
|
|
} items with a total of ${groupedChangesArray.reduce(
|
|
(acc, [_, changes]) => acc + changes.length,
|
|
0,
|
|
)} files to process`,
|
|
);
|
|
if (parallelizationFactor > 1) {
|
|
log.info(`Parallelizing ${parallelizationFactor} changes at a time`);
|
|
}
|
|
|
|
// Create a pool of workers that processes items as they become available
|
|
const pool = new Set();
|
|
const queue = [...groupedChangesArray];
|
|
// Cache git branch at the start to avoid repeated execSync calls per change
|
|
const cachedBranchForPush = opts.branch || (isGitRepository() ? getCurrentGitBranch() : null);
|
|
|
|
while (queue.length > 0 || pool.size > 0) {
|
|
// Fill the pool until we reach parallelizationFactor
|
|
while (pool.size < parallelizationFactor && queue.length > 0) {
|
|
let [_basePath, changes] = queue.shift()!;
|
|
const promise = (async () => {
|
|
const alreadySynced: string[] = [];
|
|
const isRawApp = isRawAppFile(changes[0].path);
|
|
if (isRawApp) {
|
|
const deleteRawApp = changes.find(
|
|
(change) =>
|
|
change.name === "deleted" &&
|
|
isRawAppFolderMetadataFile(change.path),
|
|
);
|
|
if (deleteRawApp) {
|
|
changes = [deleteRawApp];
|
|
} else {
|
|
changes.splice(1, changes.length - 1);
|
|
}
|
|
}
|
|
|
|
for await (const change of changes) {
|
|
let stateTarget = undefined;
|
|
if (stateful) {
|
|
try {
|
|
stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
|
|
await Deno.stat(stateTarget);
|
|
} catch {
|
|
stateTarget = undefined;
|
|
}
|
|
}
|
|
|
|
if (change.name === "edited") {
|
|
if (
|
|
await handleScriptMetadata(
|
|
change.path,
|
|
workspace,
|
|
alreadySynced,
|
|
opts.message,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
opts,
|
|
)
|
|
) {
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
continue;
|
|
} else if (
|
|
await handleFile(
|
|
change.path,
|
|
workspace,
|
|
alreadySynced,
|
|
opts.message,
|
|
opts,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
)
|
|
) {
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
continue;
|
|
}
|
|
if (stateTarget) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
log.info(
|
|
`Editing ${getTypeStrFromPath(change.path)} ${change.path}`,
|
|
);
|
|
}
|
|
|
|
if (isFileResource(change.path)) {
|
|
const resourceFilePath = await findResourceFile(change.path);
|
|
if (!alreadySynced.includes(resourceFilePath)) {
|
|
alreadySynced.push(resourceFilePath);
|
|
|
|
const newObj = parseFromPath(
|
|
resourceFilePath,
|
|
await Deno.readTextFile(resourceFilePath),
|
|
);
|
|
|
|
// For branch-specific resources, push to the base path on the workspace server
|
|
// This ensures branch-specific files are stored with their base names in the workspace
|
|
let serverPath = resourceFilePath;
|
|
const currentBranch = cachedBranchForPush;
|
|
|
|
if (currentBranch && isBranchSpecificFile(resourceFilePath)) {
|
|
serverPath = fromBranchSpecificPath(
|
|
resourceFilePath,
|
|
currentBranch,
|
|
);
|
|
}
|
|
|
|
await pushResource(
|
|
workspace.workspaceId,
|
|
serverPath,
|
|
undefined,
|
|
newObj,
|
|
resourceFilePath,
|
|
);
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
const oldObj = parseFromPath(change.path, change.before);
|
|
const newObj = parseFromPath(change.path, change.after);
|
|
|
|
// Check if this is a branch-specific item and get the original branch-specific path
|
|
let originalBranchSpecificPath: string | undefined;
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
originalBranchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
);
|
|
}
|
|
|
|
await pushObj(
|
|
workspace.workspaceId,
|
|
change.path,
|
|
oldObj,
|
|
newObj,
|
|
opts.plainSecrets ?? false,
|
|
alreadySynced,
|
|
opts.message,
|
|
originalBranchSpecificPath,
|
|
);
|
|
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.after);
|
|
}
|
|
} else if (change.name === "added") {
|
|
if (
|
|
change.path.endsWith(".script.json") ||
|
|
change.path.endsWith(".script.yaml") ||
|
|
change.path.endsWith(".lock") ||
|
|
isFileResource(change.path)
|
|
) {
|
|
continue;
|
|
} else if (
|
|
await handleFile(
|
|
change.path,
|
|
workspace,
|
|
alreadySynced,
|
|
opts.message,
|
|
opts,
|
|
rawWorkspaceDependencies,
|
|
codebases,
|
|
)
|
|
) {
|
|
continue;
|
|
}
|
|
if (stateTarget) {
|
|
await ensureDir(path.dirname(stateTarget));
|
|
log.info(
|
|
`Adding ${getTypeStrFromPath(change.path)} ${change.path}`,
|
|
);
|
|
}
|
|
const obj = parseFromPath(change.path, change.content);
|
|
|
|
// Determine the actual local file path for this change
|
|
// For branch-specific items, we read from branch-specific files but push to base server paths
|
|
let localFilePath = change.path;
|
|
if (specificItems && isSpecificItem(change.path, specificItems)) {
|
|
const branchSpecificPath = getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
);
|
|
if (branchSpecificPath) {
|
|
localFilePath = branchSpecificPath;
|
|
}
|
|
}
|
|
|
|
await pushObj(
|
|
workspace.workspaceId,
|
|
change.path,
|
|
undefined,
|
|
obj,
|
|
opts.plainSecrets ?? false,
|
|
[],
|
|
opts.message,
|
|
localFilePath, // Pass the actual local file path
|
|
);
|
|
|
|
if (stateTarget) {
|
|
await Deno.writeTextFile(stateTarget, change.content);
|
|
}
|
|
} else if (change.name === "deleted") {
|
|
if (change.path.endsWith(".lock")) {
|
|
continue;
|
|
}
|
|
const typ = getTypeStrFromPath(change.path);
|
|
|
|
if (typ == "script") {
|
|
log.info(`Archiving ${typ} ${change.path}`);
|
|
} else {
|
|
log.info(`Deleting ${typ} ${change.path}`);
|
|
}
|
|
const workspaceId = workspace.workspaceId;
|
|
const target = change.path.replaceAll(SEP, "/");
|
|
switch (typ) {
|
|
case "script": {
|
|
await wmill.archiveScriptByPath({
|
|
workspace: workspaceId,
|
|
path: removeExtensionToPath(target),
|
|
});
|
|
break;
|
|
}
|
|
case "folder":
|
|
await wmill.deleteFolder({
|
|
workspace: workspaceId,
|
|
name: change.path.split(SEP)[1],
|
|
});
|
|
break;
|
|
case "resource":
|
|
await wmill.deleteResource({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".resource.json"),
|
|
});
|
|
break;
|
|
case "resource-type":
|
|
await wmill.deleteResourceType({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".resource-type.json"),
|
|
});
|
|
break;
|
|
case "flow":
|
|
await wmill.deleteFlowByPath({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, getDeleteSuffix("flow", "json")),
|
|
});
|
|
break;
|
|
case "app":
|
|
await wmill.deleteApp({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, getDeleteSuffix("app", "json")),
|
|
});
|
|
break;
|
|
case "raw_app":
|
|
if (isRawAppFolderMetadataFile(target)) {
|
|
// Delete the entire raw app
|
|
await wmill.deleteApp({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, getDeleteSuffix("raw_app", "json")),
|
|
});
|
|
} else {
|
|
// For individual file deletions within a raw app,
|
|
// re-push the entire raw app so the backend gets the updated file list
|
|
// (the deleted file won't be included in the push)
|
|
await pushObj(
|
|
workspaceId,
|
|
target,
|
|
undefined,
|
|
undefined,
|
|
opts.plainSecrets ?? false,
|
|
alreadySynced,
|
|
opts.message,
|
|
);
|
|
}
|
|
break;
|
|
case "schedule":
|
|
await wmill.deleteSchedule({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".schedule.json"),
|
|
});
|
|
break;
|
|
case "http_trigger":
|
|
await wmill.deleteHttpTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".http_trigger.json"),
|
|
});
|
|
break;
|
|
case "websocket_trigger":
|
|
await wmill.deleteWebsocketTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".websocket_trigger.json"),
|
|
});
|
|
break;
|
|
case "kafka_trigger":
|
|
await wmill.deleteKafkaTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".kafka_trigger.json"),
|
|
});
|
|
break;
|
|
case "nats_trigger":
|
|
await wmill.deleteNatsTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".nats_trigger.json"),
|
|
});
|
|
break;
|
|
case "postgres_trigger":
|
|
await wmill.deletePostgresTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".postgres_trigger.json"),
|
|
});
|
|
break;
|
|
case "mqtt_trigger":
|
|
await wmill.deleteMqttTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".mqtt_trigger.json"),
|
|
});
|
|
break;
|
|
case "sqs_trigger":
|
|
await wmill.deleteSqsTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".sqs_trigger.json"),
|
|
});
|
|
break;
|
|
case "gcp_trigger":
|
|
await wmill.deleteGcpTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".gcp_trigger.json"),
|
|
});
|
|
break;
|
|
case "email_trigger":
|
|
await wmill.deleteEmailTrigger({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".email_trigger.json"),
|
|
});
|
|
break;
|
|
case "native_trigger": {
|
|
const triggerInfo = extractNativeTriggerInfo(change.path);
|
|
if (!triggerInfo) {
|
|
throw new Error(
|
|
`Invalid native trigger path: ${change.path}`
|
|
);
|
|
}
|
|
await wmill.deleteNativeTrigger({
|
|
workspace: workspaceId,
|
|
serviceName: triggerInfo.serviceName as NativeServiceName,
|
|
externalId: triggerInfo.externalId,
|
|
});
|
|
break;
|
|
}
|
|
case "variable":
|
|
await wmill.deleteVariable({
|
|
workspace: workspaceId,
|
|
path: removeSuffix(target, ".variable.json"),
|
|
});
|
|
break;
|
|
case "user": {
|
|
const users = await wmill.listUsers({
|
|
workspace: workspaceId,
|
|
});
|
|
|
|
const email = removeSuffix(
|
|
removePathPrefix(change.path, "users"),
|
|
".user.json",
|
|
);
|
|
const user = users.find((u) => u.email === email);
|
|
if (!user) {
|
|
throw new Error(`User ${email} not found`);
|
|
}
|
|
await wmill.deleteUser({
|
|
workspace: workspaceId,
|
|
username: user.username,
|
|
});
|
|
break;
|
|
}
|
|
case "group":
|
|
await wmill.deleteGroup({
|
|
workspace: workspaceId,
|
|
name: removeSuffix(
|
|
removePathPrefix(change.path, "groups"),
|
|
".group.json",
|
|
),
|
|
});
|
|
break;
|
|
case "workspace_dependencies":
|
|
const relativePath = removePathPrefix(
|
|
change.path,
|
|
"dependencies",
|
|
);
|
|
|
|
const res = workspaceDependenciesPathToLanguageAndFilename(
|
|
change.path,
|
|
);
|
|
if (!res) {
|
|
throw new Error(
|
|
`Unknown workspace dependencies file format: ${change.path}`,
|
|
);
|
|
}
|
|
const { name, language } = res;
|
|
|
|
await wmill.deleteWorkspaceDependencies({
|
|
workspace: workspaceId,
|
|
language,
|
|
name,
|
|
});
|
|
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
if (stateTarget) {
|
|
try {
|
|
await Deno.remove(stateTarget);
|
|
} catch {
|
|
// state target may not exist already
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})();
|
|
|
|
pool.add(promise);
|
|
// Remove from pool when complete
|
|
promise.then(() => pool.delete(promise));
|
|
}
|
|
|
|
// Wait for at least one task to complete before continuing
|
|
if (pool.size > 0) {
|
|
await Promise.race(pool);
|
|
}
|
|
}
|
|
if (opts.jsonOutput) {
|
|
const result = {
|
|
success: true,
|
|
message: `All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name}`,
|
|
changes: changes.map((change) => ({
|
|
type: change.name,
|
|
path: change.path,
|
|
...(change.name === "edited" && change.codebase
|
|
? { codebase_changed: true }
|
|
: {}),
|
|
...(specificItems && isSpecificItem(change.path, specificItems)
|
|
? {
|
|
branch_specific: true,
|
|
branch_specific_path: getBranchSpecificPath(
|
|
change.path,
|
|
specificItems,
|
|
opts.branch,
|
|
),
|
|
}
|
|
: {}),
|
|
})),
|
|
total: changes.length,
|
|
duration_ms: Math.round(performance.now() - start),
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
} else {
|
|
log.info(
|
|
colors.bold.green.underline(
|
|
`\nDone! All ${
|
|
changes.length
|
|
} changes pushed to the remote workspace ${
|
|
workspace.workspaceId
|
|
} named ${workspace.name} (${(performance.now() - start).toFixed(
|
|
0,
|
|
)}ms)`,
|
|
),
|
|
);
|
|
}
|
|
} else if (opts.jsonOutput) {
|
|
console.log(
|
|
JSON.stringify(
|
|
{ success: true, message: "No changes to push", total: 0 },
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
const command = new Command()
|
|
.description(
|
|
"sync local with a remote workspaces or the opposite (push or pull)",
|
|
)
|
|
.action(() =>
|
|
log.info("2 actions available, pull and push. Use -h to display help."),
|
|
)
|
|
.command("pull")
|
|
.description("Pull any remote changes and apply them locally.")
|
|
.option("--yes", "Pull without needing confirmation")
|
|
.option(
|
|
"--dry-run",
|
|
"Show changes that would be pulled without actually pushing",
|
|
)
|
|
.option("--plain-secrets", "Pull secrets as plain text")
|
|
.option("--json", "Use JSON instead of YAML")
|
|
.option("--skip-variables", "Skip syncing variables (including secrets)")
|
|
.option("--skip-secrets", "Skip syncing only secrets variables")
|
|
.option("--skip-resources", "Skip syncing resources")
|
|
.option("--skip-resource-types", "Skip syncing resource types")
|
|
.option("--skip-scripts", "Skip syncing scripts")
|
|
.option("--skip-flows", "Skip syncing flows")
|
|
.option("--skip-apps", "Skip syncing apps")
|
|
.option("--skip-folders", "Skip syncing folders")
|
|
.option(
|
|
"--skip-workspace-dependencies",
|
|
"Skip syncing workspace dependencies",
|
|
)
|
|
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
|
|
.option("--include-schedules", "Include syncing schedules")
|
|
.option("--include-triggers", "Include syncing triggers")
|
|
.option("--include-users", "Include syncing users")
|
|
.option("--include-groups", "Include syncing groups")
|
|
.option("--include-settings", "Include syncing workspace settings")
|
|
.option("--include-key", "Include workspace encryption key")
|
|
.option("--skip-branch-validation", "Skip git branch validation and prompts")
|
|
.option("--json-output", "Output results in JSON format")
|
|
.option(
|
|
"-i --includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes",
|
|
)
|
|
.option(
|
|
"-e --excludes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to NOT take into account. Overrides wmill.yaml excludes",
|
|
)
|
|
.option(
|
|
"--extra-includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy",
|
|
)
|
|
.option(
|
|
"--repository <repo:string>",
|
|
"Specify repository path (e.g., u/user/repo) when multiple repositories exist",
|
|
)
|
|
.option(
|
|
"--promotion <branch:string>",
|
|
"Use promotionOverrides from the specified branch instead of regular overrides",
|
|
)
|
|
.option(
|
|
"--branch <branch:string>",
|
|
"Override the current git branch (works even outside a git repository)",
|
|
)
|
|
// deno-lint-ignore no-explicit-any
|
|
.action(pull as any)
|
|
.command("push")
|
|
.description("Push any local changes and apply them remotely.")
|
|
.option("--yes", "Push without needing confirmation")
|
|
.option(
|
|
"--dry-run",
|
|
"Show changes that would be pushed without actually pushing",
|
|
)
|
|
.option("--plain-secrets", "Push secrets as plain text")
|
|
.option("--json", "Use JSON instead of YAML")
|
|
.option("--skip-variables", "Skip syncing variables (including secrets)")
|
|
.option("--skip-secrets", "Skip syncing only secrets variables")
|
|
.option("--skip-resources", "Skip syncing resources")
|
|
.option("--skip-resource-types", "Skip syncing resource types")
|
|
.option("--skip-scripts", "Skip syncing scripts")
|
|
.option("--skip-flows", "Skip syncing flows")
|
|
.option("--skip-apps", "Skip syncing apps")
|
|
.option("--skip-folders", "Skip syncing folders")
|
|
.option(
|
|
"--skip-workspace-dependencies",
|
|
"Skip syncing workspace dependencies",
|
|
)
|
|
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
|
|
.option("--include-schedules", "Include syncing schedules")
|
|
.option("--include-triggers", "Include syncing triggers")
|
|
.option("--include-users", "Include syncing users")
|
|
.option("--include-groups", "Include syncing groups")
|
|
.option("--include-settings", "Include syncing workspace settings")
|
|
.option("--include-key", "Include workspace encryption key")
|
|
.option("--skip-branch-validation", "Skip git branch validation and prompts")
|
|
.option("--json-output", "Output results in JSON format")
|
|
.option(
|
|
"-i --includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)",
|
|
)
|
|
.option(
|
|
"-e --excludes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to NOT take into account.",
|
|
)
|
|
.option(
|
|
"--extra-includes <patterns:file[]>",
|
|
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy",
|
|
)
|
|
.option(
|
|
"--message <message:string>",
|
|
"Include a message that will be added to all scripts/flows/apps updated during this push",
|
|
)
|
|
.option("--parallel <number>", "Number of changes to process in parallel")
|
|
.option(
|
|
"--repository <repo:string>",
|
|
"Specify repository path (e.g., u/user/repo) when multiple repositories exist",
|
|
)
|
|
.option(
|
|
"--branch <branch:string>",
|
|
"Override the current git branch (works even outside a git repository)",
|
|
)
|
|
.option("--lint", "Run lint validation before pushing")
|
|
// deno-lint-ignore no-explicit-any
|
|
.action(push as any);
|
|
|
|
export default command;
|