fix(cli): add --plain-secrets

backend/parsers/windmill-parser-py/src/lib.rs

@@ -181,15 +181,13 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "git" => "GitPython",
     "u" => "requests",
     "f" => "requests",
+    "." => "requests",
     "shopify" => "ShopifyAPI",
     "seleniumwire" => "selenium-wire",
     "openbb-terminal" => "openbb[all]",
 };
 
 fn replace_import(x: String) -> String {
-    if x.starts_with('.') {
-        return "requests".to_string();
-    }
     PYTHON_IMPORTS_REPLACEMENT
         .get(&x)
         .map(|x| x.to_owned())
@@ -238,8 +236,8 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
                             .map(replace_import)
                             .collect::<Vec<String>>(),
                     ),
-                    StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => {
-                        let imprt = if mod_.starts_with('.') {
+                    StmtKind::ImportFrom { level, module: Some(mod_), names: _ } => {
+                        let imprt = if level.is_some() && level.unwrap() > 0 {
                             ".".to_string()
                         } else {
                             mod_.split('.').next().unwrap_or("").replace("_", "-")
@@ -463,7 +461,7 @@ def main():
 ";
         let r = parse_python_imports(code)?;
         // println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
+        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib", "requests"]);
         Ok(())
     }
 

backend/windmill-api/src/workspaces.rs

@@ -18,6 +18,7 @@ use crate::{
     resources::{Resource, ResourceType},
     users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK, VALID_USERNAME},
     utils::require_super_admin,
+    variables::build_crypt,
     HTTP_CLIENT,
 };
 #[cfg(feature = "enterprise")]
@@ -30,6 +31,7 @@ use axum::{
     routing::{delete, get, post},
     Json, Router,
 };
+use magic_crypt::MagicCryptTrait;
 #[cfg(feature = "enterprise")]
 use stripe::CustomerId;
 use windmill_audit::{audit_log, ActionKind};
@@ -1140,6 +1142,7 @@ impl ArchiveImpl {
 #[derive(Deserialize)]
 struct ArchiveQueryParams {
     archive_type: Option<String>,
+    plain_secret: Option<bool>,
 }
 
 #[inline]
@@ -1185,7 +1188,7 @@ async fn tarball_workspace(
     authed: Authed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
-    Query(ArchiveQueryParams { archive_type }): Query<ArchiveQueryParams>,
+    Query(ArchiveQueryParams { archive_type, plain_secret }): Query<ArchiveQueryParams>,
 ) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
     require_admin(authed.is_admin, &authed.username)?;
 
@@ -1322,7 +1325,15 @@ async fn tarball_workspace(
         .fetch_all(&db)
         .await?;
 
-        for var in variables {
+        let mc = build_crypt(&mut db.begin().await?, &w_id).await?;
+
+        for mut var in variables {
+            if plain_secret.unwrap_or(false) && var.value.is_some() {
+                var.value = Some(
+                    mc.decrypt_base64_to_string(var.value.unwrap())
+                        .map_err(|e| Error::InternalErr(e.to_string()))?,
+                );
+            }
             let var_str = &to_string_without_metadata(&var, false).unwrap();
             archive
                 .write_to_archive(&var_str, &format!("{}.variable.json", var.path))

cli/pull.ts

@@ -4,7 +4,8 @@ import { colors, Command, JSZip } from "./deps.ts";
 import { Workspace } from "./workspace.ts";
 
 export async function downloadZip(
-  workspace: Workspace
+  workspace: Workspace,
+  plainWorkspace: boolean
 ): Promise<JSZip | undefined> {
   const requestHeaders: HeadersInit = new Headers();
   requestHeaders.set("Authorization", "Bearer " + workspace.token);
@@ -14,7 +15,8 @@ export async function downloadZip(
     workspace.remote +
       "api/w/" +
       workspace.workspaceId +
-      "/workspaces/tarball?archive_type=zip",
+      "/workspaces/tarball?archive_type=zip&plain_secret=" +
+      plainWorkspace,
     {
       headers: requestHeaders,
       method: "GET",

cli/sync.ts

@@ -255,7 +255,12 @@ async function ignoreF() {
 }
 
 async function pull(
-  opts: GlobalOptions & { raw: boolean; yes: boolean; failConflicts: boolean }
+  opts: GlobalOptions & {
+    raw: boolean;
+    yes: boolean;
+    failConflicts: boolean;
+    plainSecrets: boolean;
+  }
 ) {
   if (!opts.raw) {
     await ensureDir(path.join(Deno.cwd(), ".wmill"));
@@ -269,7 +274,9 @@ async function pull(
       "Computing the files to update locally to match remote (taking .wmillignore into account)"
     )
   );
-  const remote = ZipFSElement((await downloadZip(workspace))!);
+  const remote = ZipFSElement(
+    (await downloadZip(workspace, opts.plainSecrets))!
+  );
   const local = opts.raw
     ? undefined
     : await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill"));
@@ -495,6 +502,7 @@ async function push(
     yes: boolean;
     skipPull: boolean;
     failConflicts: boolean;
+    plainSecrets: boolean;
   }
 ) {
   if (!opts.raw) {
@@ -518,7 +526,7 @@ async function push(
   );
   const remote = opts.raw
     ? undefined
-    : ZipFSElement((await downloadZip(workspace))!);
+    : ZipFSElement((await downloadZip(workspace, opts.plainSecrets))!);
   const local = await FSFSElement(path.join(Deno.cwd(), ""));
   const changes = await compareDynFSElement(local, remote, await ignoreF());
 
@@ -587,7 +595,8 @@ async function push(
           workspace.workspaceId,
           change.path.split(".")[0],
           obj,
-          diff
+          diff,
+          opts.plainSecrets
         );
         if (!opts.raw && stateExists) {
           await Deno.writeTextFile(stateTarget, change.after);
@@ -617,7 +626,8 @@ async function push(
           workspace.workspaceId,
           change.path.split(".")[0],
           obj,
-          diff
+          diff,
+          opts.plainSecrets
         );
         if (!opts.raw && stateExists) {
           await Deno.writeTextFile(stateTarget, change.content);
@@ -704,7 +714,8 @@ async function push(
       | ResourceFile
       | ResourceTypeFile
       | FolderFile,
-    diffs: Difference[]
+    diffs: Difference[],
+    plainSecrets: boolean
   ) {
     if (file instanceof ScriptFile) {
       throw new Error(
@@ -723,7 +734,7 @@ async function push(
       return;
     }
     try {
-      await file.pushDiffs(workspace, remotePath, diffs);
+      await file.pushDiffs(workspace, remotePath, diffs, plainSecrets);
     } catch (e) {
       console.error("Failing to apply diffs to " + remotePath);
       console.error(JSON.stringify(e));
@@ -742,6 +753,7 @@ const command = new Command()
   )
   .option("--yes", "Pull without needing confirmation")
   .option("--raw", "Pull without using state, just overwrite.")
+  .option("--plain-secrets", "Pull secrets as plain text")
   .action(pull as any)
   .command("push")
   .description(
@@ -754,6 +766,7 @@ const command = new Command()
   .option("--skip-pull", "Push without pulling first (you have pulled prior)")
   .option("--yes", "Push without needing confirmation")
   .option("--raw", "Push without using state, just overwrite.")
+  .option("--plain-secrets", "Push secrets as plain text")
   .action(push as any);
 
 export default command;

cli/types.ts

@@ -11,7 +11,11 @@ import { AppFile } from "./apps.ts";
 // TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined.
 // Then combine those with diffing, which then gives the new push impl
 export interface Resource {
-  push(workspace: string, remotePath: string): Promise<void>;
+  push(
+    workspace: string,
+    remotePath: string,
+    plainSecrets?: boolean
+  ): Promise<void>;
 }
 
 export interface PushDiffs {
@@ -19,6 +23,7 @@ export interface PushDiffs {
     workspace: string,
     remotePath: string,
     diffs: Difference[],
+    plainSecrets?: boolean
   ): Promise<void>;
 }
 
@@ -46,7 +51,7 @@ export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange;
 export function setValueByPath(
   obj: any,
   path: (string | number)[],
-  value: any,
+  value: any
 ) {
   let i;
   let lastObj = undefined;
@@ -84,7 +89,7 @@ export type GlobalOptions = {
 
 export function inferTypeFromPath(
   p: string,
-  obj: any,
+  obj: any
 ):
   | ScriptFile
   | VariableFile
@@ -115,7 +120,7 @@ export function inferTypeFromPath(
 }
 
 export function getTypeStrFromPath(
-  p: string,
+  p: string
 ):
   | "script"
   | "variable"
@@ -125,8 +130,13 @@ export function getTypeStrFromPath(
   | "folder"
   | "app" {
   const parsed = path.parse(p);
-  if (parsed.ext == ".go" || parsed.ext == ".ts" || parsed.ext == ".sh" || parsed.ext == ".py") {
-    return 'script'
+  if (
+    parsed.ext == ".go" ||
+    parsed.ext == ".ts" ||
+    parsed.ext == ".sh" ||
+    parsed.ext == ".py"
+  ) {
+    return "script";
   }
 
   if (parsed.name === "folder.meta") {
@@ -135,9 +145,12 @@ export function getTypeStrFromPath(
 
   const typeEnding = parsed.name.split(".").at(-1);
   if (
-    typeEnding === "script" || typeEnding === "variable" ||
-    typeEnding === "flow" || typeEnding === "resource" ||
-    typeEnding === "resource-type" || typeEnding === "app"
+    typeEnding === "script" ||
+    typeEnding === "variable" ||
+    typeEnding === "flow" ||
+    typeEnding === "resource" ||
+    typeEnding === "resource-type" ||
+    typeEnding === "app"
   ) {
     return typeEnding;
   } else {

cli/variable.ts

@@ -61,7 +61,8 @@ export class VariableFile implements Resource, PushDiffs {
   async pushDiffs(
     workspace: string,
     remotePath: string,
-    diffs: Difference[]
+    diffs: Difference[],
+    plainSecrets?: boolean
   ): Promise<void> {
     if (await VariableService.existsVariable({ workspace, path: remotePath })) {
       console.log(
@@ -105,14 +106,14 @@ export class VariableFile implements Resource, PushDiffs {
       await VariableService.updateVariable({
         workspace,
         path: remotePath,
-        alreadyEncrypted: true,
+        alreadyEncrypted: !plainSecrets,
         requestBody: changeset,
       });
     } else {
       console.log(colors.yellow.bold("Creating new variable..."));
       await VariableService.createVariable({
         workspace,
-        alreadyEncrypted: true,
+        alreadyEncrypted: !plainSecrets,
         requestBody: {
           path: remotePath,
           description: this.description,
@@ -124,16 +125,25 @@ export class VariableFile implements Resource, PushDiffs {
       });
     }
   }
-  async push(workspace: string, remotePath: string): Promise<void> {
+  async push(
+    workspace: string,
+    remotePath: string,
+    plainSecrets?: boolean
+  ): Promise<void> {
     await this.pushDiffs(
       workspace,
       remotePath,
-      microdiff({}, this, { cyclesFix: false })
+      microdiff({}, this, { cyclesFix: false }),
+      plainSecrets
     );
   }
 }
 
-async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
+async function push(
+  opts: GlobalOptions & { plainSecrets: boolean },
+  filePath: string,
+  remotePath: string
+) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
@@ -148,19 +158,25 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
 
   console.log(colors.bold.yellow("Pushing variable..."));
 
-  await pushVariable(workspace.workspaceId, filePath, remotePath);
+  await pushVariable(
+    workspace.workspaceId,
+    filePath,
+    remotePath,
+    opts.plainSecrets
+  );
   console.log(colors.bold.underline.green(`Variable ${remotePath} pushed`));
 }
 
 export async function pushVariable(
   workspace: string,
   filePath: string,
-  remotePath: string
+  remotePath: string,
+  plainSecrets: boolean
 ) {
   const data = decoverto
     .type(VariableFile)
     .rawToInstance(await Deno.readTextFile(filePath));
-  await data.push(workspace, remotePath);
+  await data.push(workspace, remotePath, plainSecrets);
 }
 
 const command = new Command()
@@ -171,6 +187,7 @@ const command = new Command()
     "Push a local variable spec. This overrides any remote versions."
   )
   .arguments("<file_path:string> <remote_path:string>")
+  .option("--plain-secrets", "Push secrets as plain text")
   .action(push as any);
 
 export default command;