diff --git a/backend/migrations/20250625100238_assets.up.sql b/backend/migrations/20250625100238_assets.up.sql index 52439ef24e..26c42996fd 100644 --- a/backend/migrations/20250625100238_assets.up.sql +++ b/backend/migrations/20250625100238_assets.up.sql @@ -1,6 +1,6 @@ CREATE TYPE ASSET_USAGE_KIND AS ENUM ('script', 'flow'); CREATE TYPE ASSET_ACCESS_TYPE AS ENUM ('r', 'w', 'rw'); -CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource'); +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable'); CREATE TABLE asset ( workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index 911258c22f..d6f16824c7 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -53,23 +53,35 @@ impl AssetsFinder { .or_else(|| { node.func .as_attribute_expr() - .and_then(|attr| attr.value.as_name_expr().and_then(|o| o.id.parse().ok())) + .and_then(|attr| attr.attr.parse().ok()) }) .ok_or(())?; - let (kind, access_type) = match ident.as_str() { - "get_resource" => (AssetKind::Resource, None), - "load_s3_file" => (AssetKind::S3Object, Some(R)), - "write_s3_file" => (AssetKind::S3Object, Some(W)), + let (kind, access_type, arg) = match ident.as_str() { + "load_s3_file" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), + "load_s3_file_reader" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), + "write_s3_file" => (AssetKind::S3Object, Some(W), Arg::Pos(0)), + "get_resource" => (AssetKind::Resource, None, Arg::Pos(0)), + "set_resource" => (AssetKind::Resource, Some(W), Arg::Named("path")), + "get_boto3_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), + "get_polars_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), + "get_duckdb_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), + "get_variable" => (AssetKind::Variable, Some(R), Arg::Pos(0)), + "set_variable" => (AssetKind::Variable, Some(W), Arg::Pos(0)), _ => return Err(()), }; - if node.args.len() < 1 { - return Err(()); - } + let arg_val = match arg { + Arg::Pos(i) => node.args.get(i), + Arg::Named(name) => node + .keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some(name)) + .map(|kw| &kw.value), + }; - match &node.args[0] { - Expr::Constant(ExprConstant { value: Constant::Str(value), .. }) => { + match arg_val { + Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value); self.assets .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); @@ -79,3 +91,8 @@ impl AssetsFinder { Ok(()) } } + +enum Arg { + Pos(usize), + Named(&'static str), +} diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index 7a9b053522..dcd6681f5f 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -74,17 +74,25 @@ impl AssetsFinder { Some(Expr::Member(MemberExpr { prop: MemberProp::Ident(i), .. })) => i.sym.as_str(), _ => return Err(()), }; - let (kind, access_type) = match ident { - "getResource" => (AssetKind::Resource, None), - "loadS3File" => (AssetKind::S3Object, Some(R)), - "writeS3File" => (AssetKind::S3Object, Some(W)), + let (kind, access_type, arg_pos) = match ident { + "loadS3File" => (AssetKind::S3Object, Some(R), 0), + "loadS3FileStream" => (AssetKind::S3Object, Some(R), 0), + "writeS3File" => (AssetKind::S3Object, Some(W), 0), + "getResource" => (AssetKind::Resource, None, 0), + "setResource" => (AssetKind::Resource, Some(W), 1), + "databaseUrlFromResource" => (AssetKind::Resource, None, 0), + "denoS3LightClientSettings" => (AssetKind::Resource, None, 0), + "duckdbConnectionSettings" => (AssetKind::Resource, None, 0), + "polarsConnectionSettings" => (AssetKind::Resource, None, 0), + "getVariable" => (AssetKind::Variable, Some(R), 0), + "setVariable" => (AssetKind::Variable, Some(W), 0), _ => return Err(()), }; - if node.args.len() < 1 { - return Err(()); - } - match node.args[0].expr.as_ref() { - Expr::Lit(Lit::Str(Str { value, .. })) => { + + let arg_value = node.args.get(arg_pos); + + match arg_value.map(|e| e.expr.as_ref()) { + Some(Expr::Lit(Lit::Str(Str { value, .. }))) => { let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value); self.assets .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 6857e8d68e..893843342b 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -15,6 +15,7 @@ use AssetUsageAccessType::*; pub enum AssetKind { S3Object, Resource, + Variable, } #[derive(Serialize)] @@ -56,6 +57,8 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { Some((AssetKind::Resource, &s[6..])) } else if s.starts_with("$res:") { Some((AssetKind::Resource, &s[5..])) + } else if s.starts_with("var://") { + Some((AssetKind::Variable, &s[6..])) } else { None } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e0ff66357a..7639a4fa5d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4953,7 +4953,8 @@ paths: /scripts_u/tokened_raw/{workspace}/{token}/{path}: get: - summary: raw script by path with a token (mostly used by lsp to be used with + summary: + raw script by path with a token (mostly used by lsp to be used with import maps to resolve scripts) operationId: rawScriptByPathTokened tags: @@ -6754,7 +6755,8 @@ paths: schema: type: string - name: branch_or_iteration_n - description: for branchall or loop, the iteration at which the flow should + description: + for branchall or loop, the iteration at which the flow should restart required: true in: path @@ -8472,7 +8474,7 @@ paths: text/plain: schema: type: string - + /w/{workspace}/openapi/download: post: summary: Download the OpenAPI v3.1 spec as a file @@ -9827,7 +9829,6 @@ paths: schema: type: string - /w/{workspace}/gcp_triggers/subscriptions/delete/{path}: delete: summary: delete gcp trigger @@ -9913,7 +9914,7 @@ paths: application/json: schema: type: string - + /w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}: get: summary: check if postgres configuration is set to logical @@ -11365,7 +11366,7 @@ paths: postgres_trigger, mqtt_trigger, gcp_trigger, - sqs_trigger + sqs_trigger, ] responses: "200": @@ -11409,7 +11410,7 @@ paths: postgres_trigger, mqtt_trigger, gcp_trigger, - sqs_trigger + sqs_trigger, ] requestBody: description: acl to add @@ -11464,7 +11465,7 @@ paths: postgres_trigger, mqtt_trigger, gcp_trigger, - sqs_trigger + sqs_trigger, ] requestBody: description: acl to add @@ -11843,7 +11844,8 @@ paths: /w/{workspace}/job_helpers/duckdb_connection_settings: post: - summary: Converts an S3 resource to the set of instructions necessary to connect + summary: + Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket operationId: duckdbConnectionSettings tags: @@ -11872,7 +11874,8 @@ paths: type: string /w/{workspace}/job_helpers/v2/duckdb_connection_settings: post: - summary: Converts an S3 resource to the set of instructions necessary to connect + summary: + Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket operationId: duckdbConnectionSettingsV2 tags: @@ -11908,7 +11911,8 @@ paths: /w/{workspace}/job_helpers/polars_connection_settings: post: - summary: Converts an S3 resource to the set of arguments necessary to connect + summary: + Converts an S3 resource to the set of arguments necessary to connect Polars to an S3 bucket operationId: polarsConnectionSettings tags: @@ -11952,7 +11956,8 @@ paths: - client_kwargs /w/{workspace}/job_helpers/v2/polars_connection_settings: post: - summary: Converts an S3 resource to the set of arguments necessary to connect + summary: + Converts an S3 resource to the set of arguments necessary to connect Polars to an S3 bucket operationId: polarsConnectionSettingsV2 tags: @@ -12029,7 +12034,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" requestBody: - description: S3 resource path to use. If empty, the S3 resource defined in the + description: + S3 resource path to use. If empty, the S3 resource defined in the workspace settings will be used required: true content: @@ -12947,11 +12953,11 @@ paths: summary: List all assets in the workspace operationId: listAssets tags: - - asset + - asset parameters: - - $ref: '#/components/parameters/WorkspaceId' + - $ref: "#/components/parameters/WorkspaceId" responses: - '200': + "200": description: all assets in the workspace content: application/json: @@ -12964,7 +12970,7 @@ paths: path: type: string kind: - $ref: '#/components/schemas/AssetKind' + $ref: "#/components/schemas/AssetKind" usages: type: array items: @@ -12974,18 +12980,18 @@ paths: path: type: string kind: - $ref: '#/components/schemas/AssetUsageKind' + $ref: "#/components/schemas/AssetUsageKind" access_type: - $ref: '#/components/schemas/AssetUsageAccessType' + $ref: "#/components/schemas/AssetUsageAccessType" /w/{workspace}/assets/list_by_usages: post: summary: List all assets used by given usages paths operationId: listAssetsByUsage tags: - - asset + - asset parameters: - - $ref: '#/components/parameters/WorkspaceId' + - $ref: "#/components/parameters/WorkspaceId" requestBody: description: list assets by usages required: true @@ -13004,9 +13010,9 @@ paths: path: type: string kind: - $ref: '#/components/schemas/AssetUsageKind' + $ref: "#/components/schemas/AssetUsageKind" responses: - '200': + "200": description: all assets used by the given usage paths, in the same order content: application/json: @@ -13021,9 +13027,9 @@ paths: path: type: string kind: - $ref: '#/components/schemas/AssetKind' + $ref: "#/components/schemas/AssetKind" access_type: - $ref: '#/components/schemas/AssetUsageAccessType' + $ref: "#/components/schemas/AssetUsageAccessType" components: securitySchemes: @@ -13171,7 +13177,8 @@ components: type: string ParentJob: name: parent_job - description: The parent job that is at the origin and responsible for the execution + description: + The parent job that is at the origin and responsible for the execution of this script if any in: query schema: @@ -13191,7 +13198,8 @@ components: type: string NewJobId: name: job_id - description: The job id to assign to the created job. if missing, job is chosen + description: + The job id to assign to the created job. if missing, job is chosen randomly using the ULID scheme. If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request) in: query @@ -13282,7 +13290,8 @@ components: format: date-time CreatedOrStartedAfter: name: created_or_started_after - description: filter on created_at for non non started job and started_at otherwise + description: + filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query schema: @@ -13290,7 +13299,8 @@ components: format: date-time CreatedOrStartedAfterCompletedJob: name: created_or_started_after_completed_jobs - description: filter on created_at for non non started job and started_at otherwise + description: + filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query schema: @@ -13298,7 +13308,8 @@ components: format: date-time CreatedOrStartedBefore: name: created_or_started_before - description: filter on created_at for non non started job and started_at otherwise + description: + filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query schema: @@ -13386,7 +13397,8 @@ components: enum: [Create, Update, Delete, Execute] JobKinds: name: job_kinds - description: filter on job kind (values 'preview', 'script', 'dependencies', 'flow') + description: + filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query schema: @@ -13481,7 +13493,19 @@ components: AIProvider: type: string - enum: [openai, azure_openai, anthropic, mistral, deepseek, googleai, groq, openrouter, togetherai, customai] + enum: + [ + openai, + azure_openai, + anthropic, + mistral, + deepseek, + googleai, + groq, + openrouter, + togetherai, + customai, + ] AIProviderModel: type: object @@ -13548,7 +13572,7 @@ components: alerts: type: array items: - $ref: '#/components/schemas/Alert' + $ref: "#/components/schemas/Alert" Script: type: object @@ -13736,7 +13760,7 @@ components: access_type: type: string enum: [r, w, rw] - + required: - path - summary @@ -13884,7 +13908,7 @@ components: "singlescriptflow", "flowscript", "flownode", - "appscript" + "appscript", ] schedule_path: type: string @@ -13991,7 +14015,7 @@ components: "singlescriptflow", "flowscript", "flownode", - "appscript" + "appscript", ] schedule_path: type: string @@ -14443,7 +14467,7 @@ components: "bytes", "dict", "datetime", - "sql" + "sql", ] - type: object properties: @@ -14483,7 +14507,7 @@ components: "bytes", "dict", "datetime", - "sql" + "sql", ] - type: object properties: @@ -14509,7 +14533,7 @@ components: "bytes", "dict", "datetime", - "sql" + "sql", ] - type: object properties: @@ -14562,7 +14586,7 @@ components: csharp, nu, java, - duckdb + duckdb, # for related places search: ADD_NEW_LANG ] @@ -15037,7 +15061,7 @@ components: - user_or_folder_regex_value - path - runnable_kind - + OpenapiV3Info: type: object properties: @@ -15617,7 +15641,6 @@ components: - delivery_type - subscription_mode - SubscriptionMode: type: string enum: @@ -15625,7 +15648,6 @@ components: - create_update description: "The mode of subscription. 'existing' means using an existing GCP subscription, while 'create_update' involves creating or updating a new subscription." - GcpTriggerData: type: object properties: @@ -15667,7 +15689,6 @@ components: required: - topic_id - DeleteGcpSubscription: type: object properties: @@ -16663,7 +16684,14 @@ components: properties: type: type: string - enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"] + enum: + [ + "S3Storage", + "AzureBlobStorage", + "AzureWorkloadIdentity", + "S3AwsOidc", + "GoogleCloudStorage", + ] s3_resource_path: type: string azure_blob_resource_path: @@ -16680,7 +16708,13 @@ components: type: type: string enum: - ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"] + [ + "S3Storage", + "AzureBlobStorage", + "AzureWorkloadIdentity", + "S3AwsOidc", + "GoogleCloudStorage", + ] s3_resource_path: type: string azure_blob_resource_path: @@ -17070,7 +17104,8 @@ components: CaptureTriggerKind: type: string - enum: [webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp] + enum: + [webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp] Capture: type: object @@ -17271,24 +17306,25 @@ components: AssetUsageKind: type: string enum: - - script - - flow + - script + - flow AssetUsageAccessType: type: string enum: - - r - - w - - rw + - r + - w + - rw AssetKind: type: string enum: - - s3object - - resource + - s3object + - resource + - variable Asset: type: object properties: path: type: string kind: - $ref: '#/components/schemas/AssetKind' + $ref: "#/components/schemas/AssetKind" required: [path, kind] diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index a7b859e9d7..6b720b368d 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -10,6 +10,7 @@ use crate::{error, scripts::ScriptLang}; pub enum AssetKind { S3Object, Resource, + Variable, } #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] @@ -75,6 +76,7 @@ impl From for AssetKind { match kind { windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, + windmill_parser::asset_parser::AssetKind::Variable => AssetKind::Variable, } } } diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 0cc4f58b76..230cbc8368 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -12,9 +12,11 @@ export type AssetWithAccessType = Asset & { access_type?: AssetUsageAccessType } export function formatAsset(asset: Asset): string { switch (asset.kind) { case 'resource': - return `$res:${asset.path}` + return `res://${asset.path}` case 's3object': return `s3://${asset.path}` + case 'variable': + return `var://${asset.path}` } } diff --git a/frontend/src/lib/components/icons/AssetGenericIcon.svelte b/frontend/src/lib/components/icons/AssetGenericIcon.svelte index 48fc3d79d0..ebc33a5150 100644 --- a/frontend/src/lib/components/icons/AssetGenericIcon.svelte +++ b/frontend/src/lib/components/icons/AssetGenericIcon.svelte @@ -1,7 +1,9 @@ + + + + + + + + + + + +