diff --git a/backend/sqlx-data.json b/backend/sqlx-data.json index cd8c36eab1..f94de922cd 100644 --- a/backend/sqlx-data.json +++ b/backend/sqlx-data.json @@ -35,6 +35,74 @@ }, "query": "SELECT app.id FROM app\n WHERE app.path = $1 AND app.workspace_id = $2" }, + "023fffd0a042a28b5be991169a506aff92f64f84e49b4c041cd369b045c31e73": { + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "path", + "ordinal": 1, + "type_info": "Varchar" + }, + { + "name": "summary", + "ordinal": 2, + "type_info": "Varchar" + }, + { + "name": "versions", + "ordinal": 3, + "type_info": "Int8Array" + }, + { + "name": "policy", + "ordinal": 4, + "type_info": "Jsonb" + }, + { + "name": "extra_perms", + "ordinal": 5, + "type_info": "Jsonb" + }, + { + "name": "value", + "ordinal": 6, + "type_info": "Jsonb" + }, + { + "name": "created_at", + "ordinal": 7, + "type_info": "Timestamptz" + }, + { + "name": "created_by", + "ordinal": 8, + "type_info": "Varchar" + } + ], + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "parameters": { + "Left": [ + "Text" + ] + } + }, + "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by from app, app_version \n WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]" + }, "0355b53b1d45955ca56b2829372ce9c656d7f0ad7b8d0709161047f0d8cdc4f4": { "describe": { "columns": [ diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 45c722385c..96145d4b22 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1092,6 +1092,10 @@ paths: - variable parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: already_encrypted + in: query + schema: + type: boolean requestBody: description: new variable required: true @@ -1133,6 +1137,10 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/Path" + - name: already_encrypted + in: query + schema: + type: boolean requestBody: description: updated variable required: true diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index 9436b88d08..3a24448830 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -209,12 +209,13 @@ async fn create_variable( Extension(user_db): Extension, Extension(webhook): Extension, Path(w_id): Path, + Query(AlreadyEncrypted { already_encrypted }): Query, Json(variable): Json, ) -> Result<(StatusCode, String)> { let mut tx = user_db.begin(&authed).await?; check_path_conflict(&mut tx, &w_id, &variable.path).await?; - let value = if variable.is_secret { + let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { let mc = build_crypt(&mut tx, &w_id).await?; encrypt(&mc, &variable.value) } else { @@ -312,12 +313,18 @@ struct EditVariable { description: Option, } +#[derive(Deserialize)] +struct AlreadyEncrypted { + already_encrypted: Option, +} + async fn update_variable( authed: Authed, Extension(user_db): Extension, Extension(webhook): Extension, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(AlreadyEncrypted { already_encrypted }): Query, Json(ns): Json, ) -> Result { use sql_builder::prelude::*; @@ -343,7 +350,7 @@ async fn update_variable( .await? .unwrap_or(false); - let value = if is_secret { + let value = if is_secret && !already_encrypted.unwrap_or(false) { let mc = build_crypt(&mut tx, &w_id).await?; encrypt(&mc, &nvalue) } else { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index f04da1e381..09883f53eb 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -9,6 +9,7 @@ use std::str::FromStr; use crate::{ + apps::AppWithLastVersion, db::{UserDB, DB}, folders::Folder, resources::{Resource, ResourceType}, @@ -24,6 +25,7 @@ use axum::{ routing::{delete, get, post}, Json, Router, }; +use serde_json::to_string_pretty; use stripe::CustomerId; use windmill_audit::{audit_log, ActionKind}; use windmill_common::{ @@ -1098,6 +1100,28 @@ struct ArchiveQueryParams { archive_type: Option, } +#[inline] +pub fn to_string_without_metadata(value: &T) -> Result +where + T: ?Sized + Serialize, +{ + let value = serde_json::to_value(value).map_err(to_anyhow)?; + value + .as_object() + .map(|obj| { + let mut obj = obj.clone(); + for key in ["workspace_id", "path", "name"] { + if obj.contains_key(key) { + obj.remove(key); + } + } + + serde_json::to_string_pretty(&obj).ok() + }) + .flatten() + .ok_or_else(|| Error::BadRequest("Impossible to serialize value".to_string())) +} + async fn tarball_workspace( authed: Authed, Extension(db): Extension, @@ -1129,7 +1153,7 @@ async fn tarball_workspace( for folder in folders { archive .write_to_archive( - &serde_json::to_string_pretty(&folder).unwrap(), + &to_string_without_metadata(&folder).unwrap(), &format!("f/{}/folder.meta.json", folder.name), ) .await?; @@ -1187,7 +1211,7 @@ async fn tarball_workspace( .await?; for resource in resources { - let resource_str = serde_json::to_string_pretty(&resource).unwrap(); + let resource_str = &to_string_without_metadata(&resource).unwrap(); archive .write_to_archive(&resource_str, &format!("{}.resource.json", resource.path)) .await?; @@ -1204,7 +1228,7 @@ async fn tarball_workspace( .await?; for resource_type in resource_types { - let resource_str = serde_json::to_string_pretty(&resource_type).unwrap(); + let resource_str = &to_string_without_metadata(&resource_type).unwrap(); archive .write_to_archive( &resource_str, @@ -1223,7 +1247,7 @@ async fn tarball_workspace( .await?; for flow in flows { - let flow_str = serde_json::to_string_pretty(&flow).unwrap(); + let flow_str = &to_string_without_metadata(&flow).unwrap(); archive .write_to_archive(&flow_str, &format!("{}.flow.json", flow.path)) .await?; @@ -1232,19 +1256,39 @@ async fn tarball_workspace( { let variables = sqlx::query_as::<_, ExportableListableVariable>( - "SELECT *, false as is_expired FROM variable WHERE workspace_id = $1 AND is_secret = false", + "SELECT *, false as is_expired FROM variable WHERE workspace_id = $1", ) .bind(&w_id) .fetch_all(&db) .await?; for var in variables { - let flow_str = serde_json::to_string_pretty(&var).unwrap(); + let flow_str = &to_string_without_metadata(&var).unwrap(); archive .write_to_archive(&flow_str, &format!("{}.variable.json", var.path)) .await?; } } + + { + let apps = sqlx::query_as!( + AppWithLastVersion, + "SELECT app.id, app.path, app.summary, app.versions, app.policy, + app.extra_perms, app_version.value, + app_version.created_at, app_version.created_by from app, app_version + WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]", + &w_id + ) + .fetch_all(&db) + .await?; + + for app in apps { + let flow_str = &to_string_pretty(&app).unwrap(); + archive + .write_to_archive(&flow_str, &format!("{}.app.json", app.path)) + .await?; + } + } archive.finish().await?; let file = tokio::fs::File::open(file_path).await?; diff --git a/cli/apps.ts b/cli/apps.ts new file mode 100644 index 0000000000..358badf2a4 --- /dev/null +++ b/cli/apps.ts @@ -0,0 +1,105 @@ +import { Any, model, property } from "./decoverto.ts"; +import { + AppService, + AppWithLastVersion, + colors, + microdiff, + Policy, +} from "./deps.ts"; +import { Difference, PushDiffs, Resource, setValueByPath } from "./types.ts"; + +@model() +export class AppFile implements Resource, PushDiffs { + @property(Any) + value: any; + @property(() => String) + summary: string; + @property(Any) + policy: Policy; + + constructor(value: string, summary: string, policy: Policy) { + this.value = value; + this.summary = summary; + this.policy = policy; + } + async pushDiffs( + workspace: string, + remotePath: string, + diffs: Difference[], + ): Promise { + if (await AppService.existsApp({ workspace, path: remotePath })) { + console.log( + colors.bold.yellow( + `Applying ${diffs.length} diffs to existing app...`, + ), + ); + const changeset: { + path?: string | undefined; + summary?: string | undefined; + value?: any; + policy?: Policy | undefined; + } = {}; + for (const diff of diffs) { + if ( + diff.type !== "REMOVE" && + ( + diff.path[0] !== "value" && diff.path[0] !== "policy" && ( + diff.path.length !== 1 || + !["path", "summary"].includes( + diff.path[0] as string, + ) + ) + ) + ) { + throw new Error("Invalid app diff with path " + diff.path); + } + if (diff.type === "CREATE" || diff.type === "CHANGE") { + setValueByPath(changeset, diff.path, diff.value); + } else if (diff.type === "REMOVE") { + setValueByPath(changeset, diff.path, null); + } + } + + const hasChanges = Object.values(changeset).some((v) => + v !== null && typeof v !== "undefined" + ); + if (!hasChanges) { + console.log(colors.yellow("! Skipping empty changeset")); + return; + } + + await AppService.updateApp({ + workspace, + path: remotePath, + requestBody: changeset, + }); + } else { + console.log(colors.yellow("Creating new app...")); + await AppService.createApp({ + workspace, + requestBody: { + path: remotePath, + policy: this.policy, + summary: this.summary, + value: this.value, + }, + }); + } + } + async push(workspace: string, remotePath: string): Promise { + let existing: AppWithLastVersion | undefined; + try { + existing = await AppService.getAppByPath({ + workspace: workspace, + path: remotePath, + }); + } catch { + existing = undefined; + } + await this.pushDiffs( + workspace, + remotePath, + microdiff(existing ?? {}, this, { cyclesFix: false }), + ); + } +} diff --git a/cli/deps.ts b/cli/deps.ts index 6c2458183c..c66704bcbe 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -16,7 +16,6 @@ export { } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts"; // std -export { Untar } from "https://deno.land/std@0.176.0/archive/untar.ts"; export * as path from "https://deno.land/std@0.176.0/path/mod.ts"; export { ensureDir } from "https://deno.land/std@0.176.0/fs/ensure_dir.ts"; export { @@ -38,3 +37,5 @@ export { default as microdiff, } from "https://deno.land/x/microdiff@v1.3.1/index.ts"; export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts"; +export { default as gitignore_parser } from "npm:gitignore-parser"; +export { default as JSZip } from "npm:jszip@3.7.1"; diff --git a/cli/flow.ts b/cli/flow.ts index a8e2af2e24..c64689cbee 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -76,6 +76,7 @@ export class FlowFile implements Resource, PushDiffs { description: this.description, // This is OpenAPIed as optional, but isn't schema: this.schema, // Same }; + const base_changeset = { ...changeset }; for (const diff of diffs) { if ( diff.type !== "REMOVE" && @@ -107,7 +108,10 @@ export class FlowFile implements Resource, PushDiffs { await FlowService.updateFlow({ workspace: workspace, path: remotePath, - requestBody: changeset, + requestBody: { + ...changeset, + ...base_changeset, + }, }); } else { console.log(colors.bold.yellow("Creating new flow...")); diff --git a/cli/folder.ts b/cli/folder.ts index 102942ac9b..4be94ac824 100644 --- a/cli/folder.ts +++ b/cli/folder.ts @@ -105,12 +105,13 @@ export class FolderFile implements Resource, PushDiffs { requestBody: changeset, }); } else { - console.log(colors.bold.yellow("Creating new folder...")); + console.log(colors.bold.yellow("Creating new folder: " + remotePath)); + console.log(this.owners, this.extra_perms) await FolderService.createFolder({ workspace: workspace, requestBody: { name: remotePath, - extra_perms: this.extra_perms, + extra_perms: Object.fromEntries(this.extra_perms?.entries() ?? []), owners: this.owners, }, }); diff --git a/cli/main.ts b/cli/main.ts index a4d1a23bd0..3d9bb57a3c 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -18,6 +18,7 @@ const VERSION = "v1.64.0"; let command: any = new Command() .name("wmill") .description("A simple CLI tool for windmill.") + .action(() => command.showHelp()) .globalOption( "--workspace ", "Specify the target workspace. This overrides the default workspace.", diff --git a/cli/pull.ts b/cli/pull.ts index 6d5d5f13da..ab1493d975 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -1,43 +1,35 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions } from "./types.ts"; -import { colors, Command, readerFromStreamReader, Untar } from "./deps.ts"; +import { colors, Command, JSZip } from "./deps.ts"; import { Workspace } from "./workspace.ts"; -export async function downloadTar( +export async function downloadZip( workspace: Workspace, -): Promise { +): Promise { const requestHeaders: HeadersInit = new Headers(); requestHeaders.set("Authorization", "Bearer " + workspace.token); requestHeaders.set("Content-Type", "application/octet-stream"); - const tarResponse = await fetch( + const zipResponse = await fetch( workspace.remote + "api/w/" + workspace.workspaceId + - "/workspaces/tarball", + "/workspaces/tarball?archive_type=zip", { headers: requestHeaders, method: "GET", }, ); - if (!tarResponse.ok) { + if (!zipResponse.ok) { console.log( colors.red( - "Failed to request tarball from API " + tarResponse.statusText, + "Failed to request tarball from API " + zipResponse.statusText, ), ); - console.log(await tarResponse.text()); + console.log(await zipResponse.text()); return undefined; } - - const streamReader = tarResponse.body?.getReader(); - if (!streamReader) { - console.log(colors.red("Failed to read tar request body")); - return undefined; - } - console.log(colors.yellow("Streaming tarball to disk...")); - const denoReader = readerFromStreamReader(streamReader); - const untar = new Untar(denoReader); - return untar; + const blob = await zipResponse.blob(); + return await JSZip.loadAsync(blob); } async function stub( diff --git a/cli/resource-type.ts b/cli/resource-type.ts index d4483039c3..474dde1d7d 100644 --- a/cli/resource-type.ts +++ b/cli/resource-type.ts @@ -60,7 +60,7 @@ export class ResourceTypeFile implements ResourceI, PushDiffs { ) { console.log( "Resource type " + remotePath + - " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?", + " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?", ); return; } diff --git a/cli/sync.ts b/cli/sync.ts index 2c12ee4d4b..8d0d607e4c 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -6,9 +6,9 @@ import { colors, Command, Confirm, - copy, ensureDir, - iterateReader, + gitignore_parser, + JSZip, microdiff, nanoid, objectHash, @@ -22,7 +22,7 @@ import { inferTypeFromPath, setValueByPath, } from "./types.ts"; -import { downloadTar } from "./pull.ts"; +import { downloadZip } from "./pull.ts"; import { FolderFile } from "./folder.ts"; import { ResourceTypeFile } from "./resource-type.ts"; import { @@ -41,15 +41,13 @@ const TrackedId = String; const CONTENT_ENCODER: cbor.Encoder = new cbor.Encoder({ pack: true }); export class Tracked { - #id: TrackedId; #parent: State; path: string; #content_cache?: string; - constructor(id: TrackedId, parent: State, path: string) { - this.#id = id; + constructor(parent: State, path: string) { this.#parent = parent; this.path = path; } @@ -59,29 +57,146 @@ export class Tracked { return this.#content_cache; } - if (!this.#parent.stateRoot) { - throw new Error("Parent uninitialized"); - } - - const f = this.#parent.contentFile(this.#id); + const f = this.#parent.stateContentFor(this.path); if (!f) { return undefined; } const data = await Deno.readFile( - path.join(this.#parent.stateRoot, ".wmill", f), + State.getInternalWmillFolder(f), ); const content = CONTENT_ENCODER.decode(data); this.#content_cache = content; return content; } - getHash(): string { - return this.#parent.hashes.get(this.#id)!; + getHash(): string | undefined { + return this.#parent.hashes.get(objectHash(this.path)); } +} - getId(): TrackedId { - return this.#id; +type DynFSElement = { + isDirectory: boolean; + path: string; + getContentBytes(): Promise; + getContentText(): Promise; + getChildren(): AsyncIterable; +}; + +async function FSFSElement(p: string): Promise { + function _internal_element( + p: string, + isDir: boolean, + ): DynFSElement { + return { + isDirectory: isDir, + path: p, + async *getChildren(): AsyncIterable { + for await (const e of Deno.readDir(p)) { + yield _internal_element(path.join(p, e.name), e.isDirectory); + } + }, + async getContentBytes(): Promise { + return await Deno.readFile(p); + }, + async getContentText(): Promise { + return await Deno.readTextFile(p); + }, + }; + } + return _internal_element(p, (await Deno.stat(p)).isDirectory); +} + +function ZipFSElement(zip: JSZip): DynFSElement { + function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement { + return { + isDirectory: false, + path: p, + // deno-lint-ignore require-yield + async *getChildren(): AsyncIterable { + throw new Error("Cannot get children of file"); + }, + async getContentBytes(): Promise { + return await f.async("uint8array"); + }, + async getContentText(): Promise { + return await f.async("text"); + }, + }; + } + function _internal_folder(p: string, zip: JSZip): DynFSElement { + return { + isDirectory: true, + path: p, + async *getChildren(): AsyncIterable { + for (const filename in zip.files) { + const file = zip.files[filename]; + const totalPath = path.join(p, filename); + if (file.dir) { + const e = zip.folder(file.name)!; + yield _internal_folder(totalPath, e); + } else { + yield _internal_file(totalPath, file); + } + } + }, + async getContentBytes(): Promise { + throw new Error("Cannot get content of folder"); + }, + async getContentText(): Promise { + throw new Error("Cannot get content of folder"); + }, + }; + } + return _internal_folder("./", zip); +} + +async function* readDirRecursiveWithIgnore( + ignore: (path: string) => boolean, + root: DynFSElement, +): AsyncGenerator< + { + path: string; + ignored: boolean; + isDirectory: boolean; + getContentBytes(): Promise; + getContentText(): Promise; + } +> { + const stack: { + path: string; + isDirectory: boolean; + ignored: boolean; + c(): AsyncIterable; + getContentBytes(): Promise; + getContentText(): Promise; + }[] = [{ + path: root.path, + ignored: ignore(root.path), + isDirectory: root.isDirectory, + c: root.getChildren, + getContentBytes(): Promise { + throw undefined; + }, + getContentText(): Promise { + throw undefined; + }, + }]; + + while (stack.length > 0) { + const e = stack.pop()!; + yield e; + if (!e.isDirectory) continue; + for await (const e2 of e.c()) { + stack.push({ + path: e2.path, + ignored: e.ignored || ignore(e2.path), + isDirectory: e2.isDirectory, + getContentBytes: e2.getContentBytes, + getContentText: e2.getContentText, + c: e2.getChildren, + }); + } } } @@ -97,85 +212,97 @@ export class State { @property(map(() => TrackedId, () => String, { shape: MapShape.Object })) contentFiles: Map; - @property(map(() => String, () => TrackedId, { shape: MapShape.Object })) - tracked: Map; - @property(() => String) workspaceId: string; @property(() => String) remoteUrl: string; - stateRoot?: string; - constructor( hashes: Map, - tracked: Map, contentFiles: Map, workspaceId: string, remoteUrl: string, ) { this.hashes = hashes; - this.tracked = tracked; this.contentFiles = contentFiles; this.workspaceId = workspaceId; this.remoteUrl = remoteUrl; } - add(path: string) { - if (this.tracked.get(path)) { - throw new Error("Cannot newly track already tracked paths"); - } else { - this.tracked.set(path, nanoid()); - } - } - - public forget(path: string) { - const id = this.tracked.get(path); - if (id) { - this.tracked.delete(path); - this.hashes.delete(id); - } - } - - public contentFile(id: TrackedId): string | undefined { - return this.contentFiles.get(id); + public stateContentFor(path: string) { + const hashOfPath = objectHash(path); + const file = this.contentFiles.get(hashOfPath); + return file; } public get(path: string): Tracked { - const id = this.tracked.get(path); - if (!id) { - throw new Error("Could not resolve path " + path); + // TODO: Normalize path + return new Tracked(this, path); + } + + public static getInternalWmillFolder(...subpath: string[]): string { + return path.join(Deno.cwd(), ".wmill", ...subpath); + } + + public async *getFiles(): AsyncGenerator<{ + localFile: string | undefined; + stateFile: Tracked; + isIgnored: boolean; + path: string; + }> { + // not sure why the auto-typing doesn't work here, see (a @types package is available...) + const ignore: { + accepts(file: string): boolean; + denies(file: string): boolean; + } = gitignore_parser.compile( + await Deno.readTextFile(".wmillignore"), + ); + const base = Deno.cwd(); + for await ( + const { ignored, path, getContentText, isDirectory } + of readDirRecursiveWithIgnore( + ignore.denies, + await FSFSElement(base), + ) + ) { + if (isDirectory || path.includes(".wmill")) { + continue; + } + const path2 = path.substring(base.length + 1); + const stateFile = this.get(path2); + let localFile: string | undefined; + try { + localFile = await getContentText(); + } catch { + localFile = undefined; + } + yield { path: path2, isIgnored: ignored, stateFile, localFile }; } - return new Tracked(id, this, path); } public async save(): Promise { - if (!this.stateRoot) { - throw new Error("Uninitialized state root"); - } const encoder = new cbor.Encoder({}); const plain = decoverto.type(State).instanceToPlain(this); const result: Uint8Array = encoder.encode(plain, {}); - await Deno.writeFile(path.join(this.stateRoot, ".wmill", "main"), result, { + await Deno.writeFile(State.getInternalWmillFolder("main"), result, { create: true, }); } - public static async loadState(dir: string): Promise { - const source = await Deno.readFile(path.join(dir, ".wmill", "main")); + public static async loadState(): Promise { + const source = await Deno.readFile(this.getInternalWmillFolder("main")); const encoder = new cbor.Encoder({}); const raw = encoder.decode(source); const state = decoverto.type(State).plainToInstance( raw, ); - state.stateRoot = dir; return Object.freeze(state); } } async function getState(opts: GlobalOptions) { - const existingState = await State.loadState(Deno.cwd()); + const existingState = await State.loadState(); const workspaceStream = await getWorkspaceStream(); const reader = workspaceStream.getReader(); @@ -203,50 +330,63 @@ async function updateStateFromRemote( state: State, callback: (filename: string) => PromiseLike | boolean, ) { - const untar = await downloadTar(workspace); - if (!untar) throw new Error("Failed to pull Tar"); + const zipDir = await downloadZip(workspace); + if (!zipDir) throw new Error("Failed to pull Zip"); - const decoder = new TextDecoder(); - for await (const entry of untar) { - const id = state.tracked.get(entry.fileName); - if (id) { - let val = ""; - for await (const e of iterateReader(entry)) { - const tmp = decoder.decode(e); - val += tmp; - } - - if (entry.fileName.endsWith(".json")) { + for await ( + const entry of readDirRecursiveWithIgnore( + (_) => false, + ZipFSElement(zipDir), + ) + ) { + if (entry.isDirectory || entry.ignored) continue; + const e = state.get(entry.path); + if (e) { + const val = await entry.getContentText(); + if (entry.path.endsWith(".json")) { const parsed = JSON.parse(val); - const typed = inferTypeFromPath(entry.fileName, parsed); + const typed = inferTypeFromPath(entry.path, parsed); - const oldHash = state.hashes.get(id); + const oldHash = e.getHash(); const newHash = objectHash(typed); if (!oldHash || oldHash !== newHash) { - if (!await callback(entry.fileName)) { + if (!await callback(entry.path)) { return; // notice that we are not saving } - state.hashes.set(id, newHash); + + try { + await Deno.stat(entry.path); + } catch { + await ensureDir(path.dirname(e.path)); + await Deno.writeTextFile(e.path, "{}"); + } + state.hashes.set(objectHash(e.path), newHash); const encoded = CONTENT_ENCODER.encode(typed); const fileName = nanoid(); await Deno.writeFile( - path.join(state.stateRoot!, ".wmill", fileName), + State.getInternalWmillFolder(fileName), encoded, { create: true }, ); - state.contentFiles.set(id, fileName); + state.contentFiles.set(objectHash(e.path), fileName); } } else { const fileName = nanoid(); await Deno.writeTextFile( - path.join(state.stateRoot!, ".wmill", fileName), + State.getInternalWmillFolder(fileName), val, { create: true }, ); - state.contentFiles.set(id, fileName); + state.contentFiles.set(objectHash(e.path), fileName); + try { + await Deno.stat(entry.path); + } catch { + await ensureDir(path.dirname(e.path)); + await Deno.writeTextFile(e.path, "tmp_file"); + } } } } @@ -262,11 +402,23 @@ async function pull( opts2.override = opts2.rawOverride; opts2.raw = undefined; opts2.rawOverride = undefined; - await pullRaw(opts2, Deno.cwd()); + await pullRaw(opts2); return; } + try { + await Deno.stat(State.getInternalWmillFolder("main")) + } catch { + console.log( + colors.yellow( + "No state file found, creating empty state folder at .wmill", + ), + ); + init() + } + const state = await getState(opts); + console.log(state); const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -282,10 +434,11 @@ async function pull( for await (const diff of diffs) { await applyDiff( diff.diff, - path.join(state.stateRoot!, diff.localPath), + path.join(Deno.cwd(), diff.localPath), ); } console.log(colors.green.underline("Done! All changes applied.")); + console.log(state); async function applyDiff(diffs: Difference[], file: string) { ensureDir(path.dirname(file)); @@ -315,18 +468,20 @@ async function pull( } async function copyNonJsonFiles(state: State): Promise { - for (const t of state.tracked.keys()) { - if (t.endsWith(".json")) { + for await ( + const { path, localFile, stateFile, isIgnored } of state.getFiles() + ) { + if (path.endsWith(".json") || isIgnored || !localFile) { continue; } - const entry = state.get(t); - const target = await Deno.open(entry.path, { create: true, write: true }); + const target = await Deno.open(stateFile.path, { + create: true, + write: true, + }); const source = await Deno.open( - path.join( - state.stateRoot!, - ".wmill", - state.contentFile(entry.getId())!, + State.getInternalWmillFolder( + state.stateContentFor(stateFile.path)!, ), { read: true, @@ -339,24 +494,18 @@ async function pull( } async function* diffState(state: State): AsyncGenerator { - for (const t of state.tracked.keys()) { - if (!t.endsWith(".json")) { + for await ( + const { path, localFile, stateFile, isIgnored } of state.getFiles() + ) { + if (!path.endsWith(".json") || isIgnored) { continue; } - const entry = state.get(t); - let fileText; - try { - fileText = await Deno.readTextFile( - path.join(state.stateRoot!, entry.path), - ); - } catch { - fileText = "{}"; - } + const fileText = localFile ?? "{}"; const old = JSON.parse(fileText); const fileHash = objectHash(old); - if (fileHash !== entry.getHash()) { - const stateContent = await entry.getContent() as any; + if (fileHash !== stateFile.getHash()) { + const stateContent = await stateFile.getContent() as any; const diff = microdiff( old, @@ -364,7 +513,7 @@ async function* diffState(state: State): AsyncGenerator { { cyclesFix: false }, ); - yield new StateDiff(entry.getId(), entry.path, diff); + yield new StateDiff(stateFile.path, diff); } } } @@ -414,7 +563,7 @@ async function push(opts: GlobalOptions & { raw: boolean }) { let fileJSON; try { fileJSON = JSON.parse( - await Deno.readTextFile(path.join(state.stateRoot!, e.path)), + await Deno.readTextFile(path.join(Deno.cwd(), e.path)), ); } catch { fileJSON = {}; @@ -444,32 +593,27 @@ async function push(opts: GlobalOptions & { raw: boolean }) { return; } - for (const p of state.tracked.keys()) { - if (!p.endsWith(".json")) continue; - const entry = state.get(p); - let fileJSON; - try { - fileJSON = JSON.parse( - await Deno.readTextFile(path.join(state.stateRoot!, entry.path)), - ); - } catch { - fileJSON = {}; - } - const file = inferTypeFromPath(entry.path, fileJSON); - const eContent = - (inferTypeFromPath(entry.path, await entry.getContent())) ?? {}; + for await ( + const { path, isIgnored, stateFile, localFile } of state.getFiles() + ) { + if (!path.endsWith(".json") || isIgnored) continue; + + const fileJSON = JSON.parse(localFile ?? "{}"); + const file = inferTypeFromPath(path, fileJSON); + const eContent = (inferTypeFromPath(path, await stateFile.getContent())) ?? + {}; const fileHash = objectHash(file); const eHash = objectHash(eContent); if (fileHash !== eHash) { - const remotePath = entry.path.split(".")[0]; - const type = getTypeStrFromPath(entry.path); + const remotePath = stateFile.path.split(".")[0]; + const type = getTypeStrFromPath(stateFile.path); if (type === "script") { // Diffing makes no sense for scripts - instead fetch parent hash & check hash again. // If hash is still missmatched - create new script as child. const typed = decoverto.type(ScriptFile).plainToInstance(file); - const contentPath = await findContentFile(entry.path); + const contentPath = await findContentFile(stateFile.path); const language = inferContentTypeFromFilePath(contentPath); const content = await Deno.readTextFile(contentPath); try { @@ -562,65 +706,21 @@ async function push(opts: GlobalOptions & { raw: boolean }) { } class StateDiff { - trackedId: TrackedId; localPath: string; diff: Difference[]; constructor( - trackedId: TrackedId, localPath: string, diff: Difference[], ) { - this.trackedId = trackedId; this.localPath = localPath; this.diff = diff; } } -async function add(opts: GlobalOptions, path: string) { - const state = await getState(opts); - - // TODO: Automatically check whether this path exists either locally or on the remote - state.add(path); - - if (path.endsWith(".script.json")) { - try { - const f = await findContentFile(path); - state.add(f); - } catch { - const workspace = await resolveWorkspace(opts); - await requireLogin(opts); - try { - const old = await ScriptService.getScriptByPath({ - workspace: workspace.workspaceId, - path: path.split(".")[0], - }); - if (old.language === "python3") { - state.add(path.replace(".script.json", ".py")); - } else if (old.language === "bash") { - state.add(path.replace(".script.json", ".sh")); - } else if (old.language === "deno") { - state.add(path.replace(".script.json", ".ts")); - } else if (old.language === "go") { - state.add(path.replace(".script.json", ".go")); - } else { - throw new Error("Remote returned invalid language?! " + old.language); - } - } catch { - throw new Error( - "Could not infer script language from local or remote. Exiting.", - ); - } - } - } - - await state.save(); -} - async function init(opts: GlobalOptions) { - const root = Deno.cwd(); try { - await Deno.mkdir(path.join(root, ".wmill")); + await Deno.mkdir(State.getInternalWmillFolder()); } catch { console.log( colors.red( @@ -630,33 +730,41 @@ async function init(opts: GlobalOptions) { return; } + await Deno.writeTextFile( + ".wmillignore", + "# Write any ignores here as you see fit. Same syntax as .gitignore.", + ); + const workspace = await resolveWorkspace(opts); const newState = new State( - new Map(), new Map(), new Map(), workspace.workspaceId, workspace.remote, ); - newState.stateRoot = root; await newState.save(); } async function pullRaw( opts: GlobalOptions & { override: boolean }, - dir: string, ) { const workspace = await resolveWorkspace(opts); - const untar = await downloadTar(workspace); - if (!untar) return; + const zipDir = await downloadZip(workspace); + if (!zipDir) return; - for await (const entry of untar) { - console.log(entry.fileName); - const filePath = path.resolve(dir, entry.fileName); - if (entry.type === "directory") { + // TODO use ZipFSElement & readDirRecursiveWithIgnore here + // TODO also remember to read content via entry methods now instead of direct I/O + for await ( + const entry of readDirRecursiveWithIgnore( + (_) => false, + ZipFSElement(zipDir), + ) + ) { + const filePath = entry.path; + if (entry.isDirectory) { await ensureDir(filePath); continue; } @@ -670,21 +778,22 @@ async function pullRaw( exists = false; } if (exists) { + if (await Deno.readTextFileSync(filePath) === await entry.getContentText()) { + continue; + } if ( !(await Confirm.prompt( "Conflict at " + - filePath + - " do you want to override the local version?", + filePath + + " do you want to override the local version?", )) ) { continue; } } } - const file = await Deno.open(filePath, { write: true, create: true }); - const len = await copy(entry, file); - await file.truncate(len); - file.close(); + console.log("Writing " + filePath) + await Deno.writeFile(filePath, await entry.getContentBytes()); } console.log(colors.green("Done. Wrote all files to disk.")); } @@ -738,7 +847,7 @@ async function pushRawFindCandidateFiles( namespaceName, path: path + "folder.meta.json", }); - } catch {} + } catch { } } while (stack.length > 0) { @@ -751,8 +860,8 @@ async function pushRawFindCandidateFiles( namespaceKind: e.name == "g" ? "group" : e.name == "u" - ? "user" - : "folder", + ? "user" + : "folder", namespaceName: namespaceName, }); } else { @@ -804,7 +913,7 @@ async function pushRaw(opts: GlobalOptions, dir?: string) { console.log( colors.blue( "Found " + (normal.length + resourceTypes.length + folders.length) + - " candidates", + " candidates", ), ); for (const resourceType of resourceTypes) { @@ -883,8 +992,8 @@ async function pushRaw(opts: GlobalOptions, dir?: string) { console.log( colors.yellow( "Found resource type file at " + - candidate.path + - " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!", + candidate.path + + " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!", ), ); continue; @@ -938,7 +1047,7 @@ async function pushRaw(opts: GlobalOptions, dir?: string) { remotePath, ); } else { - typed.push(workspace.workspaceId, remotePath); + await typed.push(workspace.workspaceId, remotePath); } } console.log(colors.underline.bold.green("Successfully Pushed all files.")); @@ -947,22 +1056,23 @@ async function pushRaw(opts: GlobalOptions, dir?: string) { const command = new Command() .command("init") .description( - "Initialize this folder as sync root for the currently selected workspace & remote.", + "Initialize this folder as sync root for the currently selected workspace & remote." + + "\nBegin by initializing state tracking using `init` & add files you want to track using `add`. `push` & `pull` will then use local state to accurately track changes required on the remote.", ) .action(init as any) - .command("add") - .description("Add a local file for tracking") - .arguments("") - .action(add as any) .command("pull") - .description("Pull any remote changes and apply them locally") + .description( + "Pull any remote changes and apply them locally. Use --raw for usage without local state tracking.", + ) .option("--raw", "Pull without using state.") .option("--raw-override", "Always override local files with remote.", { depends: ["raw"], }) .action(pull as any) .command("push") - .description("Push any local changes and apply them remotely") + .description( + "Push any local changes and apply them remotely. Use --raw for usage without local state tracking.", + ) .option("--raw", "Push without using state.") .action(push as any); diff --git a/cli/types.ts b/cli/types.ts index b02502c5f6..5ffd7e1d05 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -6,6 +6,7 @@ import { ScriptFile } from "./script.ts"; import { VariableFile } from "./variable.ts"; import { path } from "./deps.ts"; import { FolderFile } from "./folder.ts"; +import { AppFile } from "./apps.ts"; // TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined. // Then combine those with diffing, which then gives the new push impl @@ -90,7 +91,8 @@ export function inferTypeFromPath( | FlowFile | ResourceFile | ResourceTypeFile - | FolderFile { + | FolderFile + | AppFile { const typeEnding = getTypeStrFromPath(p); if (typeEnding === "folder") { @@ -105,6 +107,8 @@ export function inferTypeFromPath( return decoverto.type(ResourceFile).plainToInstance(obj); } else if (typeEnding === "resource-type") { return decoverto.type(ResourceTypeFile).plainToInstance(obj); + } else if (typeEnding === "app") { + return decoverto.type(AppFile).plainToInstance(obj); } else { throw new Error("infer type unreachable"); } @@ -112,7 +116,14 @@ export function inferTypeFromPath( export function getTypeStrFromPath( p: string, -): "script" | "variable" | "flow" | "resource" | "resource-type" | "folder" { +): + | "script" + | "variable" + | "flow" + | "resource" + | "resource-type" + | "folder" + | "app" { const parsed = path.parse(p); if (parsed.ext !== ".json") { throw new Error( @@ -128,7 +139,7 @@ export function getTypeStrFromPath( if ( typeEnding === "script" || typeEnding === "variable" || typeEnding === "flow" || typeEnding === "resource" || - typeEnding === "resource-type" + typeEnding === "resource-type" || typeEnding === "app" ) { return typeEnding; } else { diff --git a/cli/workspace.ts b/cli/workspace.ts index 6281f74ef7..e1cc6fc55b 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -191,7 +191,7 @@ export async function add( } if (!workspaceId) { - workspaceId = await Input.prompt("Enter the ID of this workspace"); + workspaceId = await Input.prompt({ message: "Enter the ID of this workspace", default: workspaceName, suggestions: [workspaceName] }); } if (!remote) { @@ -203,7 +203,13 @@ export async function add( remote = url.toString(); } catch { // not a url - remote = new URL(await Input.prompt("Enter the Remote URL")).toString(); + remote = new URL( + await Input.prompt({ + message: "Enter the Remote URL", + suggestions: ["https://app.windmill.dev/"], + default: "https://app.windmill.dev/" + }), + ).toString(); } } remote = new URL(remote).toString(); // add trailing slash in all cases! diff --git a/frontend/src/lib/components/FolderEditor.svelte b/frontend/src/lib/components/FolderEditor.svelte index 15288f090b..f197a37c7e 100644 --- a/frontend/src/lib/components/FolderEditor.svelte +++ b/frontend/src/lib/components/FolderEditor.svelte @@ -168,7 +168,8 @@ path: name, kind: 'folder', requestBody: { - owner: owner_name + owner: owner_name, + write: true } }) } else if (role == 'writer') {