fix: unify clients to use server-side interpolation to retrieve full resources

backend/.sqlx/query-7a92837ad2a1181580c1e0d059fd6bc0ece84d3c315635de52bd2b364c238a71.json

@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable\n        LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "account",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_expired",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "is_secret",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      null,
+      false,
+      false
+    ]
+  },
+  "hash": "7a92837ad2a1181580c1e0d059fd6bc0ece84d3c315635de52bd2b364c238a71"
+}

backend/Cargo.lock

@@ -7129,6 +7129,7 @@ dependencies = [
  "anyhow",
  "argon2",
  "async-oauth2",
+ "async-recursion",
  "async-stripe",
  "async_zip",
  "axum",

backend/windmill-api/Cargo.toml

@@ -72,4 +72,5 @@ rsmq_async.workspace = true
 regex.workspace = true
 bytes.workspace = true
 mail-send.workspace = true
-samael = { workspace = true, optional = true }
+samael = { workspace = true, optional = true }
+async-recursion.workspace = true

backend/windmill-api/openapi.yaml

@@ -1444,6 +1444,23 @@ paths:
               schema:
                 $ref: "#/components/schemas/ListableVariable"
 
+  /w/{workspace}/variables/get_value/{path}:
+    get:
+      summary: get variable value
+      operationId: getVariableValue
+      tags:
+        - variable
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: variable
+          content:
+            application/json:
+              schema:
+                type: string
+
   /w/{workspace}/variables/exists/{path}:
     get:
       summary: does variable exists at path
@@ -1854,6 +1871,23 @@ paths:
               schema:
                 $ref: "#/components/schemas/Resource"
 
+  /w/{workspace}/resources/get_value_interpolated/{path}:
+    get:
+      summary:
+        get resource interpolated (variables and resources are fully unrolled)
+      operationId: getResourceValueInterpolated
+      tags:
+        - resource
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: resource value
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/resources/get_value/{path}:
     get:
       summary: get resource value

backend/windmill-api/src/oauth2.rs

@@ -509,7 +509,7 @@ async fn refresh_token(
 ) -> error::Result<String> {
     let tx = user_db.begin(&authed).await?;
 
-    _refresh_token(tx, &path, w_id, id).await?;
+    _refresh_token(tx, &path, &w_id, id).await?;
 
     Ok(format!("Token at path {path} refreshed"))
 }
@@ -517,7 +517,7 @@ async fn refresh_token(
 pub async fn _refresh_token<'c>(
     mut tx: Transaction<'c, Postgres>,
     path: &str,
-    w_id: String,
+    w_id: &str,
     id: i32,
 ) -> error::Result<String> {
     let account = sqlx::query!(

backend/windmill-api/src/resources.rs

@@ -18,6 +18,7 @@ use axum::{
 };
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
@@ -32,6 +33,10 @@ pub fn workspaced_service() -> Router {
         .route("/get/*path", get(get_resource))
         .route("/exists/*path", get(exists_resource))
         .route("/get_value/*path", get(get_resource_value))
+        .route(
+            "/get_value_interpolated/*path",
+            get(get_resource_value_interpolated),
+        )
         .route("/update/*path", post(update_resource))
         .route("/update_value/*path", post(update_resource_value))
         .route("/delete/*path", delete(delete_resource))
@@ -238,6 +243,84 @@ async fn get_resource_value(
     Ok(Json(value))
 }
 
+async fn get_resource_value_interpolated(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<Option<serde_json::Value>> {
+    let path = path.to_path();
+    let mut tx = user_db.clone().begin(&authed).await?;
+
+    let value_o = sqlx::query_scalar!(
+        "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+    tx.commit().await?;
+
+    let value = not_found_if_none(value_o, "Resource", path)?;
+    if let Some(value) = value {
+        Ok(Json(Some(
+            transform_json_value(&authed, &user_db, &w_id, value).await?,
+        )))
+    } else {
+        Ok(Json(None))
+    }
+}
+
+use async_recursion::async_recursion;
+
+#[async_recursion]
+pub async fn transform_json_value<'c>(
+    authed: &Authed,
+    user_db: &UserDB,
+    workspace: &str,
+    v: Value,
+) -> Result<Value> {
+    match v {
+        Value::String(y) if y.starts_with("$var:") => {
+            let path = y.strip_prefix("$var:").unwrap();
+            let tx: Transaction<'_, Postgres> = user_db.clone().begin(&authed).await?;
+            let v =
+                crate::variables::get_value_internal(tx, workspace, path, &authed.username).await?;
+            Ok(Value::String(v))
+        }
+        Value::String(y) if y.starts_with("$res:") => {
+            let path = y.strip_prefix("$res:").unwrap();
+            if path.split("/").count() < 2 {
+                return Err(Error::InternalErr(format!("Invalid resource path: {path}")));
+            }
+            let mut tx: Transaction<'_, Postgres> = user_db.clone().begin(&authed).await?;
+            let v = sqlx::query_scalar!(
+                "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+                path,
+                &workspace
+            )
+            .fetch_optional(&mut *tx)
+            .await?;
+            tx.commit().await?;
+            let v = not_found_if_none(v, "Resource", path)?;
+            if let Some(v) = v {
+                transform_json_value(authed, user_db, workspace, v).await
+            } else {
+                Ok(Value::Null)
+            }
+        }
+        Value::Object(mut m) => {
+            for (a, b) in m.clone().into_iter() {
+                m.insert(
+                    a.clone(),
+                    transform_json_value(authed, user_db, workspace, b).await?,
+                );
+            }
+            Ok(Value::Object(m))
+        }
+        a @ _ => Ok(a),
+    }
+}
+
 async fn check_path_conflict<'c>(
     tx: &mut Transaction<'c, Postgres>,
     w_id: &str,
@@ -262,7 +345,7 @@ async fn check_path_conflict<'c>(
 
 #[derive(Deserialize)]
 struct CreateResourceQuery {
-    update_if_exists: Option<bool>
+    update_if_exists: Option<bool>,
 }
 async fn create_resource(
     authed: Authed,
@@ -281,7 +364,7 @@ async fn create_resource(
     if !update_if_exists {
         check_path_conflict(&mut tx, &w_id, &resource.path).await?;
     }
-    
+
     sqlx::query!(
         "INSERT INTO resource
             (workspace_id, path, value, description, resource_type)

backend/windmill-api/src/variables.rs

@@ -41,6 +41,7 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_variables))
         .route("/list_contextual", get(list_contextual_variables))
         .route("/get/*path", get(get_variable))
+        .route("/get_value/*path", get(get_value))
         .route("/exists/*path", get(exists_variable))
         .route("/update/*path", post(update_variable))
         .route("/delete/*path", delete(delete_variable))
@@ -143,7 +144,7 @@ async fn get_variable(
         let value = variable.value.unwrap_or_else(|| "".to_string());
         ListableVariable {
             value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
-                Some(_refresh_token(tx, &variable.path, w_id, variable.account.unwrap()).await?)
+                Some(_refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await?)
             } else if !value.is_empty() && decrypt_secret {
                 let mc = build_crypt(&mut tx, &w_id).await?;
                 tx.commit().await?;
@@ -164,6 +165,63 @@ async fn get_variable(
     Ok(Json(r))
 }
 
+async fn get_value(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<String> {
+    let path = path.to_path();
+    let tx = user_db.begin(&authed).await?;
+    return get_value_internal(tx, &w_id, &path, &authed.username)
+        .await
+        .map(Json);
+}
+
+pub async fn get_value_internal<'c>(
+    mut tx: Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+    username: &str,
+) -> Result<String> {
+    let variable_o = sqlx::query!(
+        "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable
+        LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+
+    let variable = not_found_if_none(variable_o, "Variable", &path)?;
+
+    let r = if variable.is_secret {
+        audit_log(
+            &mut *tx,
+            username,
+            "variables.decrypt_secret",
+            ActionKind::Execute,
+            &w_id,
+            Some(&variable.path),
+            None,
+        )
+        .await?;
+        let value = variable.value;
+        if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
+            _refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await?
+        } else if !value.is_empty() {
+            let mc = build_crypt(&mut tx, &w_id).await?;
+            tx.commit().await?;
+
+            mc.decrypt_base64_to_string(value)
+                .map_err(|e| Error::InternalErr(e.to_string()))?
+        } else {
+            "".to_string()
+        }
+    } else {
+        variable.value
+    };
+
+    Ok(r)
+}
+
 async fn exists_variable(
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,

backend/windmill-common/src/lib.rs

@@ -172,7 +172,12 @@ pub async fn get_latest_deployed_hash_for_path<'c>(
 
     let script = utils::not_found_if_none(r_o, "script", script_path)?;
 
-    Ok((scripts::ScriptHash(script.hash), script.tag, script.concurrent_limit, script.concurrency_time_window_s))
+    Ok((
+        scripts::ScriptHash(script.hash),
+        script.tag,
+        script.concurrent_limit,
+        script.concurrency_time_window_s,
+    ))
 }
 
 pub async fn get_latest_hash_for_path<'c>(
@@ -192,5 +197,10 @@ pub async fn get_latest_hash_for_path<'c>(
 
     let script = utils::not_found_if_none(r_o, "script", script_path)?;
 
-    Ok((scripts::ScriptHash(script.hash), script.tag, script.concurrent_limit, script.concurrency_time_window_s))
+    Ok((
+        scripts::ScriptHash(script.hash),
+        script.tag,
+        script.concurrent_limit,
+        script.concurrency_time_window_s,
+    ))
 }

backend/windmill-worker/src/js_eval.rs

@@ -311,9 +311,9 @@ async fn op_variable(
     if let Some(client) = client {
         let result = client
             .get_client()
-            .get_variable(&client.workspace, path, None)
+            .get_variable_value(&client.workspace, path)
             .await?;
-        Ok(result.into_inner().value.unwrap_or_else(|| "".to_owned()))
+        Ok(result.into_inner())
     } else {
         anyhow::bail!("No client found in op state");
     }
@@ -370,7 +370,7 @@ async fn op_resource(
     if let Some(client) = client {
         let result = client
             .get_client()
-            .get_resource_value(&client.workspace, path)
+            .get_resource_value_interpolated(&client.workspace, path)
             .await?;
         Ok(result.into_inner())
     } else {

backend/windmill-worker/src/worker.rs

@@ -1263,14 +1263,11 @@ pub async fn transform_json_value(
     match v {
         Value::String(y) if y.starts_with("$var:") => {
             let path = y.strip_prefix("$var:").unwrap();
-            let v = client.get_client()
-                .get_variable(workspace, path, Some(true))
+            client.get_client()
+                .get_variable_value(workspace, path)
                 .await
                 .map_err(|_| Error::NotFound(format!("Variable {path} not found for `{name}`")))
-                .map(|v| v.into_inner())?
-                .value
-                .unwrap_or_else(|| String::new());
-            Ok(Value::String(v))
+                .map(|v| json!(v.into_inner()))
         }
         Value::String(y) if y.starts_with("$res:") => {
             let path = y.strip_prefix("$res:").unwrap();
@@ -1279,12 +1276,11 @@ pub async fn transform_json_value(
                     "Argument `{name}` is an invalid resource path: {path}",
                 )));
             }
-            let v = client.get_client()
-                .get_resource_value(workspace, path)
+            Ok(client.get_client()
+                .get_resource_value_interpolated(workspace, path)
                 .await
                 .map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))?
-                .into_inner();
-            transform_json_value(name, client, workspace, v).await
+                .into_inner())
         }
         Value::Object(mut m) => {
             for (a, b) in m.clone().into_iter() {

cli/hub.ts

@@ -3,6 +3,7 @@ import { Command, ResourceService, log } from "./deps.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
 import { pushResourceType } from "./resource-type.ts";
 import { GlobalOptions } from "./types.ts";
+import { deepEqual } from "./utils.ts";
 
 async function pull(opts: GlobalOptions) {
   const workspace = await resolveWorkspace(opts);
@@ -63,10 +64,13 @@ async function pull(opts: GlobalOptions) {
   for (const x of list) {
     if (
       resourceTypes.find(
-        (y) => y.name === x.name && typeof y.schema !== "string"
+        (y) =>
+          y.name === x.name &&
+          typeof y.schema !== "string" &&
+          deepEqual(y.schema, x.schema)
       )
     ) {
-      log.info("skipping " + x.name);
+      log.info("skipping " + x.name + " (same as current)");
       continue;
     }
     log.info("syncing " + x.name);

frontend/src/lib/components/EditorBar.svelte

@@ -282,8 +282,7 @@
 			editor.insertAtCursor(`v, _ := wmill.GetVariable("${path}")`)
 		} else if (lang == 'bash') {
 			editor.insertAtCursor(`curl -s -H "Authorization: Bearer $WM_TOKEN" \\
-  "$BASE_INTERNAL_URL/api/w/$WM_WORKSPACE/variables/get/${path}" \\
-  | jq -r .value`)
+  "$BASE_INTERNAL_URL/api/w/$WM_WORKSPACE/variables/get_value/${path}"`)
 		}
 		sendUserToast(`${name} inserted at cursor`)
 	}}
@@ -340,8 +339,8 @@
 			editor.insertAtCursor(`r, _ := wmill.GetResource("${path}")`)
 		} else if (lang == 'bash') {
 			editor.insertAtCursor(`curl -s -H "Authorization: Bearer $WM_TOKEN" \\
-  "$BASE_INTERNAL_URL/api/w/$WM_WORKSPACE/resources/get/${path}" \\
-  | jq -r .value`)
+  "$BASE_INTERNAL_URL/api/w/$WM_WORKSPACE/resources/get_value_interpolated/${path}" \\
+  | jq`)
 		}
 		sendUserToast(`${path} inserted at cursor`)
 	}}

go-client/windmill.go

@@ -41,16 +41,14 @@ func GetVariable(path string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	res, err := client.Client.GetVariableWithResponse(context.Background(), client.Workspace, path, &api.GetVariableParams{
-		DecryptSecret: newBool(true),
-	})
+	res, err := client.Client.GetVariableValueWithResponse(context.Background(), client.Workspace, path)
 	if res.StatusCode()/100 != 2 {
 		return "", errors.New(string(res.Body))
 	}
 	if err != nil {
 		return "", err
 	}
-	return *res.JSON200.Value, nil
+	return *res.JSON200, nil
 }
 
 func GetResource(path string) (interface{}, error) {
@@ -58,14 +56,14 @@ func GetResource(path string) (interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
-	res, err := client.Client.GetResourceWithResponse(context.Background(), client.Workspace, path)
+	res, err := client.Client.GetResourceValueInterpolatedWithResponse(context.Background(), client.Workspace, path)
 	if res.StatusCode()/100 != 2 {
 		return nil, errors.New(string(res.Body))
 	}
 	if err != nil {
 		return nil, err
 	}
-	return *res.JSON200.Value, nil
+	return *res.JSON200, nil
 }
 
 func SetResource(path string, value interface{}) error {

python-client/wmill/wmill/client.py

@@ -24,10 +24,6 @@ mysql = TypeAlias
 bigquery = TypeAlias
 
 
-VAR_RESOURCE_PREFIX = "$var:"
-RES_RESOURCE_PREFIX = "$res:"
-
-
 class JobStatus(Enum):
     WAITING = 1
     RUNNING = 2
@@ -160,7 +156,9 @@ def get_resource(path: str | None = None, none_if_undefined: bool = False) -> An
     """
     Returns the resource at a given path
     """
-    from windmill_api.api.resource import get_resource as get_resource_api
+    from windmill_api.api.resource import (
+        get_resource_value_interpolated as get_resource_api,
+    )
 
     path = path or get_state_path()
     parsed = get_resource_api.sync_detailed(
@@ -174,11 +172,7 @@ def get_resource(path: str | None = None, none_if_undefined: bool = False) -> An
                 f"Resource at path {path} does not exist or you do not have read permissions on it"
             )
 
-    if isinstance(parsed.value, Unset):
-        return None
-
-    raw = parsed.value
-    return _transform_leaf(raw)
+    return parsed
 
 
 def whoami() -> WhoamiResponse200 | None:
@@ -286,7 +280,7 @@ def get_variable(path: str) -> str:
     """
     Returns the variable at a given path as a string
     """
-    from windmill_api.api.variable import get_variable as get_variable_api
+    from windmill_api.api.variable import get_variable_value as get_variable_api
 
     res = get_variable_api.sync_detailed(
         workspace=get_workspace(), path=path, client=create_client()
@@ -296,7 +290,7 @@ def get_variable(path: str) -> str:
             f"Variable at path {path} does not exist or you do not have read permissions on it"
         )
 
-    return res.value  # type: ignore
+    return res
 
 
 def set_variable(path: str, value: str) -> None:
@@ -356,23 +350,3 @@ def get_resume_urls(approver: str | None = None) -> Dict:
         return res.parsed.to_dict()
     else:
         raise Exception("Failed to get resume urls")
-
-
-def _transform_leaves(d: Dict[str, Any]) -> Dict[str, Any]:
-    return {k: _transform_leaf(v) for k, v in d.items()}
-
-
-def _transform_leaf(v: Any) -> Any:
-    if isinstance(v, dict):
-        return _transform_leaves(v)  # type: ignore
-    elif isinstance(v, str):
-        if v.startswith(VAR_RESOURCE_PREFIX):
-            var_name = v[len(VAR_RESOURCE_PREFIX) :]
-            return get_variable(var_name)
-        if v.startswith(RES_RESOURCE_PREFIX):
-            res_name = v[len(RES_RESOURCE_PREFIX) :]
-            return get_resource(res_name)
-        else:
-            return v
-    else:
-        return v

typescript-client/client.ts

@@ -72,8 +72,10 @@ export async function getResource(
   const workspace = getWorkspace();
   path = path ?? getStatePath();
   try {
-    const resource = await ResourceService.getResource({ workspace, path });
-    return await _transformLeaf(resource.value);
+    return await ResourceService.getResourceValueInterpolated({
+      workspace,
+      path,
+    });
   } catch (e: any) {
     if (undefinedIfEmpty && e.status === 404) {
       return undefined;
@@ -96,31 +98,6 @@ export async function resolveDefaultResource(obj: any): Promise<any> {
   }
 }
 
-/**
- * Get the full resource value by path
- * @param path path of the resource,  default to internal state path
- * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error
- * @returns full resource
- */
-export async function getFullResource(
-  path?: string,
-  undefinedIfEmpty?: boolean
-): Promise<any> {
-  const workspace = getWorkspace();
-  path = path ?? getStatePath();
-  try {
-    const resource = await ResourceService.getResource({ workspace, path });
-    const value = await _transformLeaf(resource.value);
-    return { ...resource, value };
-  } catch (e: any) {
-    if (undefinedIfEmpty && e.status === 404) {
-      return undefined;
-    } else {
-      throw Error(`Resource not found at ${path} or not visible to you`);
-    }
-  }
-}
-
 export function getStatePath(): string {
   const state_path = getEnv("WM_STATE_PATH_NEW");
   if (state_path === undefined) {
@@ -217,12 +194,11 @@ export async function getState(): Promise<any> {
  * @param path path of the variable
  * @returns variable value
  */
-export async function getVariable(path: string): Promise<string | undefined> {
+export async function getVariable(path: string): Promise<string> {
   !clientSet && setClient();
   const workspace = getWorkspace();
   try {
-    const variable = await VariableService.getVariable({ workspace, path });
-    return variable.value;
+    return await VariableService.getVariableValue({ workspace, path });
   } catch (e: any) {
     throw Error(`Variable not found at ${path} or not visible to you`);
   }
@@ -262,31 +238,6 @@ export async function setVariable(
   }
 }
 
-async function transformLeaves(d: {
-  [key: string]: any;
-}): Promise<{ [key: string]: any }> {
-  for (const k in d) {
-    d[k] = await _transformLeaf(d[k]);
-  }
-  return d;
-}
-
-const VAR_RESOURCE_PREFIX = "$var:";
-const RES_RESOURCE_PREFIX = "$res:";
-async function _transformLeaf(v: any): Promise<any> {
-  if (typeof v === "object") {
-    return transformLeaves(v);
-  } else if (typeof v === "string" && v.startsWith(VAR_RESOURCE_PREFIX)) {
-    const varName = v.substring(VAR_RESOURCE_PREFIX.length);
-    return await getVariable(varName);
-  } else if (typeof v === "string" && v.startsWith(RES_RESOURCE_PREFIX)) {
-    const resName = v.substring(RES_RESOURCE_PREFIX.length);
-    return await getResource(resName);
-  } else {
-    return v;
-  }
-}
-
 export async function databaseUrlFromResource(path: string): Promise<string> {
   const resource = await getResource(path);
   return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`;