From bd7586a5eec5516fe291070303fa6516d8adc8de Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 11 Mar 2025 19:38:55 +0100 Subject: [PATCH] fix: codebases compatible with git sync (#5470) --- README.md | 4 +- backend/src/main.rs | 26 ++- backend/windmill-api/src/workspaces_export.rs | 4 +- cli/codebase.ts | 32 +-- cli/conf.ts | 1 + cli/metadata.ts | 2 +- cli/pull.ts | 22 +- cli/script.ts | 61 +++-- cli/sync.ts | 212 ++++++++++-------- 9 files changed, 202 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index e2c9a27aa6..a733727825 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,7 @@ you to have it being synced automatically everyday. ## Environment Variables | Environment Variable name | Default | Description | Api Server/Worker/All | -| ----------------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | --- | ------ | +| ----------------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | | DATABASE_URL | | The Postgres database url. | All | | WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker | | MODE | standalone | The mode if the binary. Possible values: standalone, worker, server, agent | All | @@ -352,7 +352,7 @@ you to have it being synced automatically everyday. | GO_PATH | /usr/bin/go | The path to the go binary. | Worker | | GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker | | GOPROXY | | The GOPROXY env variable to use | Worker | -| NETRC | | The netrc content to use a private go registry | Worker | | Worker | +| NETRC | | The netrc content to use a private go registry | Worker | | PY_CONCURRENT_DOWNLOADS | 20 | Sets the maximum number of in-flight concurrent python downloads that windmill will perform at any given time. | Worker | | PATH | None | The path environment variable, usually inherited | Worker | | HOME | None | The home directory to use for Go and Bash , usually inherited | Worker | diff --git a/backend/src/main.rs b/backend/src/main.rs index 3795a157ec..33d562b1dd 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -659,17 +659,6 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("received killpill for monitor job"); break; }, - _ = tokio::time::sleep(Duration::from_secs(30)) => { - monitor_db( - &db, - &base_internal_url, - server_mode, - worker_mode, - false, - tx.clone(), - ) - .await; - }, notification = listener.recv() => { match notification { Ok(n) => { @@ -857,7 +846,20 @@ Windmill Community Edition {GIT_VERSION} } } }; - } + }, + _ = tokio::time::sleep(Duration::from_secs(30)) => { + tracing::info!("monitor task started"); + monitor_db( + &db, + &base_internal_url, + server_mode, + worker_mode, + false, + tx.clone(), + ) + .await; + tracing::info!("monitor task finished"); + }, } } }); diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index f2c574de45..9cb02fc567 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -147,6 +147,7 @@ pub(crate) struct ArchiveQueryParams { skip_secrets: Option, skip_variables: Option, skip_resources: Option, + skip_resource_types: Option, include_schedules: Option, include_triggers: Option, include_users: Option, @@ -280,6 +281,7 @@ pub(crate) async fn tarball_workspace( plain_secret, plain_secrets, skip_resources, + skip_resource_types, skip_secrets, skip_variables, include_schedules, @@ -428,7 +430,7 @@ pub(crate) async fn tarball_workspace( } } - if !skip_resources.unwrap_or(false) { + if !skip_resource_types.unwrap_or(false) { let resource_types = sqlx::query_as!( ResourceType, "SELECT * FROM resource_type WHERE workspace_id = $1", diff --git a/cli/codebase.ts b/cli/codebase.ts index 844a51f52d..8af46ff284 100644 --- a/cli/codebase.ts +++ b/cli/codebase.ts @@ -2,25 +2,31 @@ import { Codebase, SyncOptions } from "./conf.ts"; import { log } from "./deps.ts"; import { digestDir } from "./utils.ts"; -export type SyncCodebase = Codebase & { digest: string }; -export async function listSyncCodebases( +export type SyncCodebase = Codebase & { getDigest: () => Promise }; +export function listSyncCodebases( options: SyncOptions -): Promise { +): SyncCodebase[] { const res: SyncCodebase[] = []; const nb_codebase = options?.codebases?.length ?? 0; if (nb_codebase > 0) { - log.info(`Found ${nb_codebase} codebases:`); + log.info(`Found ${nb_codebase} codebases: ${options?.codebases?.map((c) => c.relative_path).join(", ")}`); } for (const codebase of options?.codebases ?? []) { - let digest = await digestDir( - codebase.relative_path, - JSON.stringify(codebase) - ); - if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { - digest += ".tar"; - } - log.info(`Codebase ${codebase.relative_path}, digest: ${digest}`); - res.push({ ...codebase, digest }); + let _digest: string | undefined = undefined; + const getDigest: () => Promise = async () => { + if (_digest == undefined) { + _digest = await digestDir( + codebase.relative_path, + JSON.stringify(codebase) + ); + if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + _digest += ".tar"; + } + log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); + } + return _digest; + }; + res.push({ ...codebase, getDigest }); } return res; diff --git a/cli/conf.ts b/cli/conf.ts index a8598d5d0b..b2b7b64f54 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -10,6 +10,7 @@ export interface SyncOptions { json?: boolean; skipVariables?: boolean; skipResources?: boolean; + skipResourceTypes?: boolean; skipSecrets?: boolean; includeSchedules?: boolean; includeTriggers?: boolean; diff --git a/cli/metadata.ts b/cli/metadata.ts index 8bfb21a7cc..ac0dde6bd5 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -98,7 +98,7 @@ function findClosestRawReqs( const TOP_HASH = "__flow_hash"; async function generateFlowHash(folder: string) { - const elems = await FSFSElement(path.join(Deno.cwd(), folder), []); + const elems = await FSFSElement(path.join(Deno.cwd(), folder), [], true); const hashes: Record = {}; for await (const f of elems.getChildren()) { if (exts.some((e) => f.path.endsWith(e))) { diff --git a/cli/pull.ts b/cli/pull.ts index 51474f9eb8..f3286d5b17 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -9,6 +9,7 @@ export async function downloadZip( plainSecrets: boolean | undefined, skipVariables?: boolean, skipResources?: boolean, + skipResourceTypes?: boolean, skipSecrets?: boolean, includeSchedules?: boolean, includeTriggers?: boolean, @@ -31,19 +32,14 @@ export async function downloadZip( const zipResponse = await fetch( workspace.remote + - "api/w/" + - workspace.workspaceId + - `/workspaces/tarball?archive_type=zip&plain_secret=${ - plainSecrets ?? false - }&skip_variables=${skipVariables ?? false}&skip_resources=${ - skipResources ?? false - }&skip_secrets=${skipSecrets ?? false}&include_schedules=${ - includeSchedules ?? false - }&include_triggers=${includeTriggers ?? false}&include_users=${ - includeUsers ?? false - }&include_groups=${includeGroups ?? false}&include_settings=${ - includeSettings ?? false - }&include_key=${includeKey ?? false}&default_ts=${defaultTs ?? "bun"}`, + "api/w/" + + workspace.workspaceId + + `/workspaces/tarball?archive_type=zip&plain_secret=${plainSecrets ?? false + }&skip_variables=${skipVariables ?? false}&skip_resources=${skipResources ?? false + }&skip_secrets=${skipSecrets ?? false}&include_schedules=${includeSchedules ?? false + }&include_triggers=${includeTriggers ?? false}&include_users=${includeUsers ?? false + }&include_groups=${includeGroups ?? false}&include_settings=${includeSettings ?? false + }&include_key=${includeKey ?? false}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}`, { headers: requestHeaders, method: "GET", diff --git a/cli/script.ts b/cli/script.ts index 2d6295b50f..2a46ad842b 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -119,7 +119,7 @@ export async function findResourceFile(path: string) { if (validCandidates.length > 1) { throw new Error( "Found two resource files for the same resource" + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { @@ -249,20 +249,20 @@ export async function handleFile( let typed = opts?.skipScriptsMetadata ? undefined : ( - await parseMetadataFile( - remotePath, - opts - ? { - ...opts, - path, - workspaceRemote: workspace, - schemaOnly: codebase ? true : undefined, - } - : undefined, - globalDeps, - codebases - ) - )?.payload; + await parseMetadataFile( + remotePath, + opts + ? { + ...opts, + path, + workspaceRemote: workspace, + schemaOnly: codebase ? true : undefined, + } + : undefined, + globalDeps, + codebases + ) + )?.payload; const workspaceId = workspace.workspaceId; @@ -300,7 +300,7 @@ export async function handleFile( } if (typed && codebase) { - typed.codebase = codebase.digest; + typed.codebase = await codebase.getDigest(); } const requestBodyCommon: NewScript = { @@ -325,8 +325,7 @@ export async function handleFile( has_preprocessor: typed?.has_preprocessor, priority: typed?.priority, concurrency_key: typed?.concurrency_key, - //@ts-ignore - codebase: codebase?.digest, + codebase: await codebase?.getDigest(), timeout: typed?.timeout, on_behalf_of_email: typed?.on_behalf_of_email, }; @@ -348,19 +347,19 @@ export async function handleFile( deepEqual(typed.schema, remote.schema) && typed.tag == remote.tag && (typed.ws_error_handler_muted ?? false) == - remote.ws_error_handler_muted && + remote.ws_error_handler_muted && typed.dedicated_worker == remote.dedicated_worker && typed.cache_ttl == remote.cache_ttl && typed.concurrency_time_window_s == - remote.concurrency_time_window_s && + remote.concurrency_time_window_s && typed.concurrent_limit == remote.concurrent_limit && Boolean(typed.restart_unless_cancelled) == - Boolean(remote.restart_unless_cancelled) && + Boolean(remote.restart_unless_cancelled) && Boolean(typed.visible_to_runner_only) == - Boolean(remote.visible_to_runner_only) && + Boolean(remote.visible_to_runner_only) && Boolean(typed.no_main_func) == Boolean(remote.no_main_func) && Boolean(typed.has_preprocessor) == - Boolean(remote.has_preprocessor) && + Boolean(remote.has_preprocessor) && typed.priority == Boolean(remote.priority) && typed.timeout == remote.timeout && //@ts-ignore @@ -451,8 +450,7 @@ async function createScript( }); } catch (e: any) { throw Error( - `Script creation for ${body.path} with parent ${ - body.parent_hash + `Script creation for ${body.path} with parent ${body.parent_hash } was not successful: ${e.body ?? e.message} ` ); } @@ -478,8 +476,7 @@ async function createScript( }); if (req.status != 201) { throw Error( - `Script snapshot creation was not successful: ${req.status} - ${ - req.statusText + `Script snapshot creation was not successful: ${req.status} - ${req.statusText } - ${await req.text()} ` ); } @@ -491,8 +488,8 @@ export async function findContentFile(filePath: string) { const candidates = filePath.endsWith("script.json") ? exts.map((x) => filePath.replace(".script.json", x)) : filePath.endsWith("script.lock") - ? exts.map((x) => filePath.replace(".script.lock", x)) - : exts.map((x) => filePath.replace(".script.yaml", x)); + ? exts.map((x) => filePath.replace(".script.lock", x)) + : exts.map((x) => filePath.replace(".script.yaml", x)); const validCandidates = ( await Promise.all( @@ -511,7 +508,7 @@ export async function findContentFile(filePath: string) { if (validCandidates.length > 1) { throw new Error( "No content path given and more than one candidate found: " + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { @@ -864,7 +861,7 @@ export async function findGlobalDeps(): Promise { const pkgs: { [key: string]: string } = {}; const reqs: { [key: string]: string } = {}; const composers: { [key: string]: string } = {}; - const els = await FSFSElement(Deno.cwd(), []); + const els = await FSFSElement(Deno.cwd(), [], false); for await (const entry of readDirRecursiveWithIgnore((p, isDir) => { p = SEP + p; return ( @@ -927,7 +924,7 @@ async function generateMetadata( } else { const ignore = await ignoreF(opts); const elems = await elementsToMap( - await FSFSElement(Deno.cwd(), codebases), + await FSFSElement(Deno.cwd(), codebases, false), (p, isD) => { return ( (!isD && !exts.some((ext) => p.endsWith(ext))) || diff --git a/cli/sync.ts b/cli/sync.ts index 55f5852960..622eaae2d2 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -94,7 +94,8 @@ export function findCodebase( async function addCodebaseDigestIfRelevant( path: string, content: string, - codebases: SyncCodebase[] + codebases: SyncCodebase[], + ignoreCodebaseChanges: boolean ): Promise { const isScript = path.endsWith(".script.yaml"); if (!isScript) { @@ -115,7 +116,11 @@ async function addCodebaseDigestIfRelevant( if (c) { const parsed: any = yamlParseContent(path, content); if (parsed && typeof parsed == "object") { - parsed["codebase"] = c.digest; + if (ignoreCodebaseChanges) { + parsed["codebase"] = undefined + } else { + parsed["codebase"] = await c.getDigest(); + } parsed["lock"] = ""; return yamlStringify(parsed, yamlOptions); } else { @@ -130,7 +135,8 @@ async function addCodebaseDigestIfRelevant( export async function FSFSElement( p: string, - codebases: SyncCodebase[] + codebases: SyncCodebase[], + ignoreCodebaseChanges: boolean ): Promise { function _internal_element( localP: string, @@ -163,7 +169,8 @@ export async function FSFSElement( const r = await addCodebaseDigestIfRelevant( itemPath, content, - codebases + codebases, + ignoreCodebaseChanges ); return r; }, @@ -345,7 +352,8 @@ function ZipFSElement( zip: JSZip, useYaml: boolean, defaultTs: "bun" | "deno", - resourceTypeToFormatExtension: Record + resourceTypeToFormatExtension: Record, + ignoreCodebaseChanges: boolean ): DynFSElement { async function _internal_file( p: string, @@ -356,12 +364,12 @@ function ZipFSElement( ) ? "flow" : p.endsWith("app.json") - ? "app" - : p.endsWith("script.json") - ? "script" - : p.endsWith("resource.json") - ? "resource" - : "other"; + ? "app" + : p.endsWith("script.json") + ? "script" + : p.endsWith("resource.json") + ? "resource" + : "other"; const isJson = p.endsWith(".json"); @@ -391,7 +399,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, s.path), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return s.content; @@ -402,7 +410,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, "flow.yaml"), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return yamlStringify(flow, yamlOptions); @@ -418,7 +426,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, s.path), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return s.content; @@ -429,7 +437,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, "app.yaml"), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return yamlStringify(app, yamlOptions); @@ -457,6 +465,9 @@ function ZipFSElement( } else { parsed["lock"] = undefined; } + if (ignoreCodebaseChanges && parsed["codebase"]) { + parsed["codebase"] = undefined; + } return useYaml ? yamlStringify(parsed, yamlOptions) : JSON.stringify(parsed, null, 2); @@ -494,7 +505,7 @@ function ZipFSElement( r.push({ isDirectory: false, path: removeSuffix(finalPath, ".json") + ".lock", - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return lock; @@ -517,7 +528,7 @@ function ZipFSElement( removeSuffix(finalPath, ".resource.json") + ".resource.file." + formatExtension, - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return fileContent; @@ -578,19 +589,19 @@ export async function* readDirRecursiveWithIgnore( // getContentBytes(): Promise; getContentText(): Promise; }[] = [ - { - path: root.path, - ignored: ignore(root.path, root.isDirectory), - isDirectory: root.isDirectory, - c: root.getChildren, - // getContentBytes(): Promise { - // throw undefined; - // }, - getContentText(): Promise { - throw undefined; + { + path: root.path, + ignored: ignore(root.path, root.isDirectory), + isDirectory: root.isDirectory, + c: root.getChildren, + // getContentBytes(): Promise { + // throw undefined; + // }, + getContentText(): Promise { + throw undefined; + }, }, - }, - ]; + ]; while (stack.length > 0) { const e = stack.pop()!; @@ -653,9 +664,10 @@ export async function elementsToMap( if (!skips.includeSettings && path === "settings" + ext) continue; if (!skips.includeKey && path === "encryption_key") continue; if (skips.skipResources && path.endsWith(".resource" + ext)) continue; + if (skips.skipResourceTypes && path.endsWith(".resource-type" + ext)) continue; + if (skips.skipVariables && path.endsWith(".variable" + ext)) continue; - if (skips.skipResources && path.endsWith(".resource" + ext)) continue; if (skips.skipResources && isFileResource(path)) continue; if ( @@ -704,6 +716,7 @@ export async function elementsToMap( export interface Skips { skipVariables?: boolean | undefined; skipResources?: boolean | undefined; + skipResourceTypes?: boolean | undefined; skipSecrets?: boolean | undefined; skipScriptsMetadata?: boolean | undefined; includeSchedules?: boolean | undefined; @@ -721,13 +734,14 @@ async function compareDynFSElement( json: boolean, skips: Skips, ignoreMetadataDeletion: boolean, - codebases: SyncCodebase[] + codebases: SyncCodebase[], + ignoreCodebaseChanges: boolean ): Promise { const [m1, m2] = els2 ? await Promise.all([ - elementsToMap(els1, ignore, json, skips), - elementsToMap(els2, ignore, json, skips), - ]) + elementsToMap(els1, ignore, json, skips), + elementsToMap(els2, ignore, json, skips), + ]) : [await elementsToMap(els1, ignore, json, skips), {}]; const changes: Change[] = []; @@ -761,7 +775,9 @@ async function compareDynFSElement( return yamlParseContent(k, v); } } + const codebaseChanges: Record = {}; + for (let [k, v] of Object.entries(m1)) { const isScriptMetadata = k.endsWith(".script.yaml") || k.endsWith(".script.json"); @@ -785,16 +801,19 @@ async function compareDynFSElement( if (deepEqual(before, after)) { continue; } - if (before.codebase != undefined) { - delete before.codebase; - m2[k] = yamlStringify(before, yamlOptions); - } - if (after.codebase != undefined) { - if (before.codebase != after.codebase) { - codebaseChanges[k] = after.codebase; + if (!ignoreCodebaseChanges) { + + if (before.codebase != undefined) { + delete before.codebase; + m2[k] = yamlStringify(before, yamlOptions); + } + if (after.codebase != undefined) { + if (before.codebase != after.codebase) { + codebaseChanges[k] = after.codebase; + } + delete after.codebase; + v = yamlStringify(after, yamlOptions); } - delete after.codebase; - v = yamlStringify(after, yamlOptions); } if (skipMetadata) { continue; @@ -810,6 +829,7 @@ async function compareDynFSElement( } } + const remoteCodebase: Record = {}; for (const [k] of Object.entries(m2)) { if (m1[k] === undefined) { @@ -827,34 +847,38 @@ async function compareDynFSElement( } } - for (const [k, v] of Object.entries(remoteCodebase)) { - const tsFile = k.replace(".script.yaml", ".ts"); - if ( - changes.find( - (c) => c.path == tsFile && (c.name == "edited" || c.name == "deleted") - ) - ) { - continue; - } - let c = findCodebase(tsFile, codebases); - if (c?.digest != v) { - changes.push({ - name: "edited", - path: tsFile, - codebase: v, - before: m1[tsFile], - after: m2[tsFile], - }); + if (!ignoreCodebaseChanges) { + for (const [k, v] of Object.entries(remoteCodebase)) { + const tsFile = k.replace(".script.yaml", ".ts"); + if ( + changes.find( + (c) => c.path == tsFile && (c.name == "edited" || c.name == "deleted") + ) + ) { + continue; + } + const c = findCodebase(tsFile, codebases); + if (await c?.getDigest() != v) { + changes.push({ + name: "edited", + path: tsFile, + codebase: v, + before: m1[tsFile], + after: m2[tsFile], + }); + } } } - for (const change of changes) { - const codebase = codebaseChanges[change.path]; - if (!codebase) continue; + if (!ignoreCodebaseChanges) { + for (const change of changes) { + const codebase = codebaseChanges[change.path]; + if (!codebase) continue; - const tsFile = change.path.replace(".script.yaml", ".ts"); - if (change.name == "edited" && change.path == tsFile) { - change.codebase = codebase; + const tsFile = change.path.replace(".script.yaml", ".ts"); + if (change.name == "edited" && change.path == tsFile) { + change.codebase = codebase; + } } } @@ -960,6 +984,8 @@ export async function ignoreF(wmillconf: { includes?: string[]; excludes?: string[]; extraIncludes?: string[]; + skipResourceTypes?: boolean; + json?: boolean; }): Promise<(p: string, isDirectory: boolean) => boolean> { let whitelist: { approve(file: string): boolean } | undefined = undefined; @@ -991,6 +1017,10 @@ export async function ignoreF(wmillconf: { // new Gitignore.default({ initialRules: ignoreContent.split("\n")}).ignoreContent).compile(); return (p: string, isDirectory: boolean) => { + const ext = wmillconf.json ? ".json" : ".yaml"; + if (!isDirectory && p.endsWith(".resource-type" + ext)) { + return wmillconf.skipResourceTypes ?? false; + } return ( !isWhitelisted(p) && (isNotWmillFile(p, isDirectory) || @@ -1082,6 +1112,7 @@ export async function pull(opts: GlobalOptions & SyncOptions) { opts.plainSecrets, opts.skipVariables, opts.skipResources, + opts.skipResourceTypes, opts.skipSecrets, opts.includeSchedules, opts.includeTriggers, @@ -1089,15 +1120,16 @@ export async function pull(opts: GlobalOptions & SyncOptions) { opts.includeGroups, opts.includeSettings, opts.includeKey, - opts.defaultTs + opts.defaultTs, ))!, !opts.json, opts.defaultTs ?? "bun", - resourceTypeToFormatExtension + resourceTypeToFormatExtension, + true ); const local = !opts.stateful - ? await FSFSElement(Deno.cwd(), codebases) - : await FSFSElement(path.join(Deno.cwd(), ".wmill"), []); + ? await FSFSElement(Deno.cwd(), codebases, true) + : await FSFSElement(path.join(Deno.cwd(), ".wmill"), [], true); const changes = await compareDynFSElement( remote, local, @@ -1105,7 +1137,8 @@ export async function pull(opts: GlobalOptions & SyncOptions) { opts.json ?? false, opts, false, - codebases + codebases, + true ); log.info( @@ -1278,8 +1311,8 @@ function prettyChanges(changes: Change[]) { log.info( colors.yellow( `~ ${getTypeStrFromPath(change.path)} ` + - change.path + - (change.codebase ? ` (codebase changed)` : "") + change.path + + (change.codebase ? ` (codebase changed)` : "") ) ); if (change.before != change.after) { @@ -1354,6 +1387,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { opts.plainSecrets, opts.skipVariables, opts.skipResources, + opts.skipResourceTypes, opts.skipSecrets, opts.includeSchedules, opts.includeTriggers, @@ -1361,14 +1395,15 @@ export async function push(opts: GlobalOptions & SyncOptions) { opts.includeGroups, opts.includeSettings, opts.includeKey, - opts.defaultTs + opts.defaultTs, ))!, !opts.json, opts.defaultTs ?? "bun", - resourceTypeToFormatExtension + resourceTypeToFormatExtension, + false ); - const local = await FSFSElement(path.join(Deno.cwd(), ""), codebases); + const local = await FSFSElement(path.join(Deno.cwd(), "",), codebases, false); const changes = await compareDynFSElement( local, remote, @@ -1376,12 +1411,12 @@ export async function push(opts: GlobalOptions & SyncOptions) { opts.json ?? false, opts, true, - codebases + codebases, + false ); const globalDeps = await findGlobalDeps(); - console.log("globalDeps", globalDeps); const tracker: ChangeTracker = await buildTracker(changes); @@ -1486,8 +1521,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { } const groupedChangesArray = Array.from(groupedChanges.entries()); log.info( - `found changes for ${ - groupedChangesArray.length + `found changes for ${groupedChangesArray.length } items with a total of ${groupedChangesArray.reduce( (acc, [_, changes]) => acc + changes.length, 0 @@ -1729,11 +1763,11 @@ export async function push(opts: GlobalOptions & SyncOptions) { }); break; case "mqtt_trigger": - await wmill.deleteMqttTrigger({ - workspace: workspaceId, - path: removeSuffix(target, ".mqtt_trigger.json"), - }); - break; + await wmill.deleteMqttTrigger({ + workspace: workspaceId, + path: removeSuffix(target, ".mqtt_trigger.json"), + }); + break; case "sqs_trigger": await wmill.deleteSqsTrigger({ workspace: workspaceId, @@ -1800,8 +1834,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { } log.info( colors.bold.green.underline( - `\nDone! All ${changes.length} changes pushed to the remote workspace ${ - workspace.workspaceId + `\nDone! All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId } named ${workspace.name} (${(performance.now() - start).toFixed(0)}ms)` ) ); @@ -1824,6 +1857,7 @@ const command = new Command() .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") .option("--skip-resources", "Skip syncing resources") + .option("--skip-resource-types", "Skip syncing resource types") // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-triggers", "Include syncing triggers") @@ -1853,6 +1887,8 @@ const command = new Command() .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") .option("--skip-resources", "Skip syncing resources") + .option("--skip-resource-types", "Skip syncing resource types") + // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-triggers", "Include syncing triggers")