From 0acbcb967818b2cb72ae44df61d00447ca259ce4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 28 Mar 2023 22:45:40 +0200 Subject: [PATCH] fix(cli): add --plain-secrets --- backend/parsers/windmill-parser-py/src/lib.rs | 10 +++--- backend/windmill-api/src/workspaces.rs | 15 ++++++-- cli/pull.ts | 6 ++-- cli/sync.ts | 27 ++++++++++---- cli/types.ts | 31 +++++++++++----- cli/variable.ts | 35 ++++++++++++++----- 6 files changed, 89 insertions(+), 35 deletions(-) diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 0099381437..4d9771258f 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -181,15 +181,13 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "git" => "GitPython", "u" => "requests", "f" => "requests", + "." => "requests", "shopify" => "ShopifyAPI", "seleniumwire" => "selenium-wire", "openbb-terminal" => "openbb[all]", }; fn replace_import(x: String) -> String { - if x.starts_with('.') { - return "requests".to_string(); - } PYTHON_IMPORTS_REPLACEMENT .get(&x) .map(|x| x.to_owned()) @@ -238,8 +236,8 @@ pub fn parse_python_imports(code: &str) -> error::Result> { .map(replace_import) .collect::>(), ), - StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => { - let imprt = if mod_.starts_with('.') { + StmtKind::ImportFrom { level, module: Some(mod_), names: _ } => { + let imprt = if level.is_some() && level.unwrap() > 0 { ".".to_string() } else { mod_.split('.').next().unwrap_or("").replace("_", "-") @@ -463,7 +461,7 @@ def main(): "; let r = parse_python_imports(code)?; // println!("{}", serde_json::to_string(&r)?); - assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]); + assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib", "requests"]); Ok(()) } diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index be4695538e..7fd349a3e0 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -18,6 +18,7 @@ use crate::{ resources::{Resource, ResourceType}, users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK, VALID_USERNAME}, utils::require_super_admin, + variables::build_crypt, HTTP_CLIENT, }; #[cfg(feature = "enterprise")] @@ -30,6 +31,7 @@ use axum::{ routing::{delete, get, post}, Json, Router, }; +use magic_crypt::MagicCryptTrait; #[cfg(feature = "enterprise")] use stripe::CustomerId; use windmill_audit::{audit_log, ActionKind}; @@ -1140,6 +1142,7 @@ impl ArchiveImpl { #[derive(Deserialize)] struct ArchiveQueryParams { archive_type: Option, + plain_secret: Option, } #[inline] @@ -1185,7 +1188,7 @@ async fn tarball_workspace( authed: Authed, Extension(db): Extension, Path(w_id): Path, - Query(ArchiveQueryParams { archive_type }): Query, + Query(ArchiveQueryParams { archive_type, plain_secret }): Query, ) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> { require_admin(authed.is_admin, &authed.username)?; @@ -1322,7 +1325,15 @@ async fn tarball_workspace( .fetch_all(&db) .await?; - for var in variables { + let mc = build_crypt(&mut db.begin().await?, &w_id).await?; + + for mut var in variables { + if plain_secret.unwrap_or(false) && var.value.is_some() { + var.value = Some( + mc.decrypt_base64_to_string(var.value.unwrap()) + .map_err(|e| Error::InternalErr(e.to_string()))?, + ); + } let var_str = &to_string_without_metadata(&var, false).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) diff --git a/cli/pull.ts b/cli/pull.ts index a64ec50e92..8752513121 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -4,7 +4,8 @@ import { colors, Command, JSZip } from "./deps.ts"; import { Workspace } from "./workspace.ts"; export async function downloadZip( - workspace: Workspace + workspace: Workspace, + plainWorkspace: boolean ): Promise { const requestHeaders: HeadersInit = new Headers(); requestHeaders.set("Authorization", "Bearer " + workspace.token); @@ -14,7 +15,8 @@ export async function downloadZip( workspace.remote + "api/w/" + workspace.workspaceId + - "/workspaces/tarball?archive_type=zip", + "/workspaces/tarball?archive_type=zip&plain_secret=" + + plainWorkspace, { headers: requestHeaders, method: "GET", diff --git a/cli/sync.ts b/cli/sync.ts index 0c0908514b..a3c6d9be0d 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -255,7 +255,12 @@ async function ignoreF() { } async function pull( - opts: GlobalOptions & { raw: boolean; yes: boolean; failConflicts: boolean } + opts: GlobalOptions & { + raw: boolean; + yes: boolean; + failConflicts: boolean; + plainSecrets: boolean; + } ) { if (!opts.raw) { await ensureDir(path.join(Deno.cwd(), ".wmill")); @@ -269,7 +274,9 @@ async function pull( "Computing the files to update locally to match remote (taking .wmillignore into account)" ) ); - const remote = ZipFSElement((await downloadZip(workspace))!); + const remote = ZipFSElement( + (await downloadZip(workspace, opts.plainSecrets))! + ); const local = opts.raw ? undefined : await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill")); @@ -495,6 +502,7 @@ async function push( yes: boolean; skipPull: boolean; failConflicts: boolean; + plainSecrets: boolean; } ) { if (!opts.raw) { @@ -518,7 +526,7 @@ async function push( ); const remote = opts.raw ? undefined - : ZipFSElement((await downloadZip(workspace))!); + : ZipFSElement((await downloadZip(workspace, opts.plainSecrets))!); const local = await FSFSElement(path.join(Deno.cwd(), "")); const changes = await compareDynFSElement(local, remote, await ignoreF()); @@ -587,7 +595,8 @@ async function push( workspace.workspaceId, change.path.split(".")[0], obj, - diff + diff, + opts.plainSecrets ); if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.after); @@ -617,7 +626,8 @@ async function push( workspace.workspaceId, change.path.split(".")[0], obj, - diff + diff, + opts.plainSecrets ); if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.content); @@ -704,7 +714,8 @@ async function push( | ResourceFile | ResourceTypeFile | FolderFile, - diffs: Difference[] + diffs: Difference[], + plainSecrets: boolean ) { if (file instanceof ScriptFile) { throw new Error( @@ -723,7 +734,7 @@ async function push( return; } try { - await file.pushDiffs(workspace, remotePath, diffs); + await file.pushDiffs(workspace, remotePath, diffs, plainSecrets); } catch (e) { console.error("Failing to apply diffs to " + remotePath); console.error(JSON.stringify(e)); @@ -742,6 +753,7 @@ const command = new Command() ) .option("--yes", "Pull without needing confirmation") .option("--raw", "Pull without using state, just overwrite.") + .option("--plain-secrets", "Pull secrets as plain text") .action(pull as any) .command("push") .description( @@ -754,6 +766,7 @@ const command = new Command() .option("--skip-pull", "Push without pulling first (you have pulled prior)") .option("--yes", "Push without needing confirmation") .option("--raw", "Push without using state, just overwrite.") + .option("--plain-secrets", "Push secrets as plain text") .action(push as any); export default command; diff --git a/cli/types.ts b/cli/types.ts index f3e0ef0645..eda52c92b2 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -11,7 +11,11 @@ import { AppFile } from "./apps.ts"; // TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined. // Then combine those with diffing, which then gives the new push impl export interface Resource { - push(workspace: string, remotePath: string): Promise; + push( + workspace: string, + remotePath: string, + plainSecrets?: boolean + ): Promise; } export interface PushDiffs { @@ -19,6 +23,7 @@ export interface PushDiffs { workspace: string, remotePath: string, diffs: Difference[], + plainSecrets?: boolean ): Promise; } @@ -46,7 +51,7 @@ export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange; export function setValueByPath( obj: any, path: (string | number)[], - value: any, + value: any ) { let i; let lastObj = undefined; @@ -84,7 +89,7 @@ export type GlobalOptions = { export function inferTypeFromPath( p: string, - obj: any, + obj: any ): | ScriptFile | VariableFile @@ -115,7 +120,7 @@ export function inferTypeFromPath( } export function getTypeStrFromPath( - p: string, + p: string ): | "script" | "variable" @@ -125,8 +130,13 @@ export function getTypeStrFromPath( | "folder" | "app" { const parsed = path.parse(p); - if (parsed.ext == ".go" || parsed.ext == ".ts" || parsed.ext == ".sh" || parsed.ext == ".py") { - return 'script' + if ( + parsed.ext == ".go" || + parsed.ext == ".ts" || + parsed.ext == ".sh" || + parsed.ext == ".py" + ) { + return "script"; } if (parsed.name === "folder.meta") { @@ -135,9 +145,12 @@ export function getTypeStrFromPath( const typeEnding = parsed.name.split(".").at(-1); if ( - typeEnding === "script" || typeEnding === "variable" || - typeEnding === "flow" || typeEnding === "resource" || - typeEnding === "resource-type" || typeEnding === "app" + typeEnding === "script" || + typeEnding === "variable" || + typeEnding === "flow" || + typeEnding === "resource" || + typeEnding === "resource-type" || + typeEnding === "app" ) { return typeEnding; } else { diff --git a/cli/variable.ts b/cli/variable.ts index 75a7323e17..22ac3a08f0 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -61,7 +61,8 @@ export class VariableFile implements Resource, PushDiffs { async pushDiffs( workspace: string, remotePath: string, - diffs: Difference[] + diffs: Difference[], + plainSecrets?: boolean ): Promise { if (await VariableService.existsVariable({ workspace, path: remotePath })) { console.log( @@ -105,14 +106,14 @@ export class VariableFile implements Resource, PushDiffs { await VariableService.updateVariable({ workspace, path: remotePath, - alreadyEncrypted: true, + alreadyEncrypted: !plainSecrets, requestBody: changeset, }); } else { console.log(colors.yellow.bold("Creating new variable...")); await VariableService.createVariable({ workspace, - alreadyEncrypted: true, + alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath, description: this.description, @@ -124,16 +125,25 @@ export class VariableFile implements Resource, PushDiffs { }); } } - async push(workspace: string, remotePath: string): Promise { + async push( + workspace: string, + remotePath: string, + plainSecrets?: boolean + ): Promise { await this.pushDiffs( workspace, remotePath, - microdiff({}, this, { cyclesFix: false }) + microdiff({}, this, { cyclesFix: false }), + plainSecrets ); } } -async function push(opts: GlobalOptions, filePath: string, remotePath: string) { +async function push( + opts: GlobalOptions & { plainSecrets: boolean }, + filePath: string, + remotePath: string +) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -148,19 +158,25 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { console.log(colors.bold.yellow("Pushing variable...")); - await pushVariable(workspace.workspaceId, filePath, remotePath); + await pushVariable( + workspace.workspaceId, + filePath, + remotePath, + opts.plainSecrets + ); console.log(colors.bold.underline.green(`Variable ${remotePath} pushed`)); } export async function pushVariable( workspace: string, filePath: string, - remotePath: string + remotePath: string, + plainSecrets: boolean ) { const data = decoverto .type(VariableFile) .rawToInstance(await Deno.readTextFile(filePath)); - await data.push(workspace, remotePath); + await data.push(workspace, remotePath, plainSecrets); } const command = new Command() @@ -171,6 +187,7 @@ const command = new Command() "Push a local variable spec. This overrides any remote versions." ) .arguments(" ") + .option("--plain-secrets", "Push secrets as plain text") .action(push as any); export default command;