diff --git a/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json b/backend/.sqlx/query-11b698f82a54aac68b3617047dfe2b18dd6da7d962118fee276af354218baac2.json similarity index 84% rename from backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json rename to backend/.sqlx/query-11b698f82a54aac68b3617047dfe2b18dd6da7d962118fee276af354218baac2.json index 65cb5c6633..9900461761 100644 --- a/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json +++ b/backend/.sqlx/query-11b698f82a54aac68b3617047dfe2b18dd6da7d962118fee276af354218baac2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\" FROM http_trigger", "describe": { "columns": [ { @@ -65,6 +65,11 @@ } } } + }, + { + "ordinal": 10, + "name": "static_asset_config: _", + "type_info": "Jsonb" } ], "parameters": { @@ -80,8 +85,9 @@ false, false, false, - false + false, + true ] }, - "hash": "fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183" + "hash": "11b698f82a54aac68b3617047dfe2b18dd6da7d962118fee276af354218baac2" } diff --git a/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json b/backend/.sqlx/query-333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1.json similarity index 76% rename from backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json rename to backend/.sqlx/query-333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1.json index df94d893a6..bb21c2a9db 100644 --- a/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json +++ b/backend/.sqlx/query-333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())", + "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now())", "describe": { "columns": [], "parameters": { @@ -27,11 +27,12 @@ } } }, + "Jsonb", "Varchar", "Varchar" ] }, "nullable": [] }, - "hash": "c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8" + "hash": "333b484ffa030dee08e7b1161fcbc48af411377d2d9f58f92fc9d5eacdf0fba1" } diff --git a/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json b/backend/.sqlx/query-487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c.json similarity index 72% rename from backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json rename to backend/.sqlx/query-487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c.json index b66edc6d73..7cd37b4c50 100644 --- a/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json +++ b/backend/.sqlx/query-487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() \n WHERE workspace_id = $11 AND path = $12", + "query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now() \n WHERE workspace_id = $12 AND path = $13", "describe": { "columns": [], "parameters": { @@ -24,6 +24,7 @@ } } }, + "Jsonb", "Varchar", "Varchar", "Bool", @@ -34,5 +35,5 @@ }, "nullable": [] }, - "hash": "4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea" + "hash": "487d377e2df67fc3ea39d183ba9f99d45828d7c8e0ff10c5d74c454472e0493c" } diff --git a/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json b/backend/.sqlx/query-7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df.json similarity index 68% rename from backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json rename to backend/.sqlx/query-7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df.json index ca0a922067..da3503cf35 100644 --- a/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json +++ b/backend/.sqlx/query-7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() \n WHERE workspace_id = $9 AND path = $10", + "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now() \n WHERE workspace_id = $10 AND path = $11", "describe": { "columns": [], "parameters": { @@ -22,6 +22,7 @@ } } }, + "Jsonb", "Varchar", "Varchar", "Bool", @@ -32,5 +33,5 @@ }, "nullable": [] }, - "hash": "55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0" + "hash": "7113d7cc72e44e4b7e01b69cc18cbe7b0399cf8ec0e9e6d2b05ceef589c432df" } diff --git a/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json b/backend/.sqlx/query-c9930fcfe79541af570eace58ba7e15a0816a6b4fd036cf7b991a210654b2633.json similarity index 83% rename from backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json rename to backend/.sqlx/query-c9930fcfe79541af570eace58ba7e15a0816a6b4fd036cf7b991a210654b2633.json index a27bc6365a..da7b44d1b9 100644 --- a/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json +++ b/backend/.sqlx/query-c9930fcfe79541af570eace58ba7e15a0816a6b4fd036cf7b991a210654b2633.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger WHERE workspace_id = $1", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\" FROM http_trigger WHERE workspace_id = $1", "describe": { "columns": [ { @@ -65,6 +65,11 @@ } } } + }, + { + "ordinal": 10, + "name": "static_asset_config: _", + "type_info": "Jsonb" } ], "parameters": { @@ -82,8 +87,9 @@ false, false, false, - false + false, + true ] }, - "hash": "02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441" + "hash": "c9930fcfe79541af570eace58ba7e15a0816a6b4fd036cf7b991a210654b2633" } diff --git a/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json b/backend/.sqlx/query-f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf.json similarity index 83% rename from backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json rename to backend/.sqlx/query-f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf.json index d954f2c26c..08592a3f04 100644 --- a/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json +++ b/backend/.sqlx/query-f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2", + "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as \"static_asset_config: _\"\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2", "describe": { "columns": [ { @@ -80,6 +80,11 @@ "ordinal": 12, "name": "requires_auth", "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "static_asset_config: _", + "type_info": "Jsonb" } ], "parameters": { @@ -101,8 +106,9 @@ false, false, false, - false + false, + true ] }, - "hash": "4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491" + "hash": "f904702536c106b0e5da8facae119c6af887c49a29ae44b3a95350ff27fb1ccf" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9923d56e66..6a33fdaadd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -aedaf655693cb9bab59fcf79a48c60aad6822668 \ No newline at end of file +6534b0f31fb4a75dd706fca2ce91e37e77e4ad02 \ No newline at end of file diff --git a/backend/migrations/20241107163553_http_route_static_asset.down.sql b/backend/migrations/20241107163553_http_route_static_asset.down.sql new file mode 100644 index 0000000000..b16c347028 --- /dev/null +++ b/backend/migrations/20241107163553_http_route_static_asset.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE http_trigger DROP COLUMN static_asset_config; \ No newline at end of file diff --git a/backend/migrations/20241107163553_http_route_static_asset.up.sql b/backend/migrations/20241107163553_http_route_static_asset.up.sql new file mode 100644 index 0000000000..fdd1a25b71 --- /dev/null +++ b/backend/migrations/20241107163553_http_route_static_asset.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE http_trigger ADD COLUMN static_asset_config JSONB; \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c97f81dfde..a66bd92019 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9146,6 +9146,14 @@ paths: in: query schema: type: string + - name: content_type + in: query + schema: + type: string + - name: content_disposition + in: query + schema: + type: string requestBody: description: File content required: true @@ -11463,6 +11471,17 @@ components: type: string route_path: type: string + static_asset_config: + type: object + properties: + s3: + type: string + storage: + type: string + filename: + type: string + required: + - s3 is_flow: type: boolean extra_perms: @@ -11509,6 +11528,17 @@ components: type: string route_path: type: string + static_asset_config: + type: object + properties: + s3: + type: string + storage: + type: string + filename: + type: string + required: + - s3 is_flow: type: boolean http_method: @@ -11542,6 +11572,17 @@ components: type: string route_path: type: string + static_asset_config: + type: object + properties: + s3: + type: string + storage: + type: string + filename: + type: string + required: + - s3 is_flow: type: boolean http_method: diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index 61757cb164..2c323c0e8c 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -1,24 +1,5 @@ -use axum::{ - extract::{Path, Query}, - response::IntoResponse, - routing::{delete, get, post}, - Extension, Json, Router, -}; -use http::{HeaderMap, StatusCode}; -use serde::{Deserialize, Serialize}; -use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::prelude::FromRow; -use std::collections::HashMap; -use tower_http::cors::CorsLayer; -use windmill_audit::{audit_ee::audit_log, ActionKind}; -use windmill_common::{ - db::UserDB, - error::{self, JsonResult}, - utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, - worker::{to_raw_value, CLOUD_HOSTED}, -}; -use windmill_queue::PushArgsOwned; - +#[cfg(feature = "parquet")] +use crate::job_helpers_ee::get_workspace_s3_resource; use crate::{ db::{ApiAuthed, DB}, jobs::{ @@ -27,6 +8,31 @@ use crate::{ }, users::{fetch_api_authed, OptAuthed}, }; +use axum::{ + extract::{Path, Query}, + response::IntoResponse, + routing::{delete, get, post}, + Extension, Json, Router, +}; +#[cfg(feature = "parquet")] +use http::header::IF_NONE_MATCH; +use http::{HeaderMap, StatusCode}; +use serde::{Deserialize, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::prelude::FromRow; +use std::collections::HashMap; +use tower_http::cors::CorsLayer; +use windmill_audit::{audit_ee::audit_log, ActionKind}; +#[cfg(feature = "parquet")] +use windmill_common::s3_helpers::build_object_store_client; +use windmill_common::{ + db::UserDB, + error::{self, JsonResult}, + s3_helpers::S3Object, + utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, + worker::{to_raw_value, CLOUD_HOSTED}, +}; +use windmill_queue::PushArgsOwned; lazy_static::lazy_static! { static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap(); @@ -99,6 +105,7 @@ struct NewTrigger { is_async: bool, requires_auth: bool, http_method: HttpMethod, + static_asset_config: Option>, } #[derive(FromRow, Serialize)] @@ -116,6 +123,7 @@ struct Trigger { is_async: bool, requires_auth: bool, http_method: HttpMethod, + static_asset_config: Option>, } #[derive(Deserialize)] @@ -127,6 +135,7 @@ struct EditTrigger { is_async: bool, requires_auth: bool, http_method: HttpMethod, + static_asset_config: Option>, } #[derive(Deserialize)] @@ -182,7 +191,7 @@ async fn get_trigger( let path = path.to_path(); let trigger = sqlx::query_as!( Trigger, - r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth + r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as "static_asset_config: _" FROM http_trigger WHERE workspace_id = $1 AND path = $2"#, w_id, @@ -209,7 +218,7 @@ async fn create_trigger( let mut tx = user_db.begin(&authed).await?; sqlx::query!( - "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())", + "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now())", w_id, ct.path, ct.route_path, @@ -218,7 +227,8 @@ async fn create_trigger( ct.is_flow, ct.is_async, ct.requires_auth, - ct.http_method as HttpMethod, + ct.http_method as _, + ct.static_asset_config as _, &authed.username, &authed.email ) @@ -261,14 +271,15 @@ async fn update_trigger( sqlx::query!( "UPDATE http_trigger - SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() - WHERE workspace_id = $11 AND path = $12", + SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now() + WHERE workspace_id = $12 AND path = $13", ct.route_path, &route_path_key, ct.script_path, ct.path, ct.is_flow, - ct.http_method as HttpMethod, + ct.http_method as _, + ct.static_asset_config as _, &authed.username, &authed.email, ct.is_async, @@ -279,12 +290,13 @@ async fn update_trigger( .execute(&mut *tx).await?; } else { sqlx::query!( - "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() - WHERE workspace_id = $9 AND path = $10", + "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now() + WHERE workspace_id = $10 AND path = $11", ct.script_path, ct.path, ct.is_flow, - ct.http_method as HttpMethod, + ct.http_method as _, + ct.static_asset_config as _, &authed.username, &authed.email, ct.is_async, @@ -405,6 +417,7 @@ struct TriggerRoute { edited_by: String, email: String, http_method: HttpMethod, + static_asset_config: Option>, } async fn get_http_route_trigger( @@ -421,7 +434,7 @@ async fn get_http_route_trigger( let route_path = StripPath(splitted.collect::>().join("/")); let triggers = sqlx::query_as!( TriggerRoute, - r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger WHERE workspace_id = $1"#, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _", static_asset_config as "static_asset_config: _" FROM http_trigger WHERE workspace_id = $1"#, w_id ) .fetch_all(db) @@ -430,7 +443,7 @@ async fn get_http_route_trigger( } else { let triggers = sqlx::query_as!( TriggerRoute, - r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger"#, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _", static_asset_config as "static_asset_config: _" FROM http_trigger"#, ) .fetch_all(db) .await?; @@ -518,6 +531,90 @@ async fn route_job( Ok(trigger) => trigger, Err(e) => return e.into_response(), }; + + #[cfg(not(feature = "parquet"))] + if trigger.static_asset_config.is_some() { + return error::Error::InternalErr( + "Static asset configuration is not supported in this build".to_string(), + ) + .into_response(); + } + + #[cfg(feature = "parquet")] + if let Some(sqlx::types::Json(config)) = trigger.static_asset_config { + let build_static_response_f = async { + let (_, s3_resource_opt) = get_workspace_s3_resource( + &authed, + &db, + None, + &"NO_TOKEN".to_string(), // no token is provided in this case + &trigger.workspace_id, + config.storage, + ) + .await?; + let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr( + "No files storage resource defined at the workspace level".to_string(), + ))?; + let s3_client = build_object_store_client(&s3_resource).await?; + let path = object_store::path::Path::from(config.s3); + let s3_object = s3_client.get(&path).await.map_err(|err| { + tracing::warn!("Error retrieving file from S3: {:?}", err); + error::Error::InternalErr(format!("Error retrieving file: {}", err.to_string())) + })?; + let mut response_headers = http::HeaderMap::new(); + if let Some(ref e_tag) = s3_object.meta.e_tag { + if let Some(if_none_match) = headers.get(IF_NONE_MATCH) { + if if_none_match == e_tag { + return Ok::<_, error::Error>(( + StatusCode::NOT_MODIFIED, + response_headers, + axum::body::Body::empty(), + )); + } + } + if let Ok(e_tag) = e_tag.parse() { + response_headers.insert("etag", e_tag); + } + } + response_headers.insert( + "content-type", + s3_object + .attributes + .get(&object_store::Attribute::ContentType) + .map(|s| s.parse().ok()) + .flatten() + .unwrap_or("application/octet-stream".parse().unwrap()), + ); + response_headers.insert( + "content-disposition", + config.filename.as_ref().map_or_else( + || { + s3_object + .attributes + .get(&object_store::Attribute::ContentDisposition) + .map(|s| s.parse().ok()) + .flatten() + .unwrap_or("inline".parse().unwrap()) + }, + |filename| { + format!("inline; filename=\"{}\"", filename) + .parse() + .unwrap_or("inline".parse().unwrap()) + }, + ), + ); + + let body_stream = axum::body::Body::from_stream(s3_object.into_stream()); + Ok::<_, error::Error>((StatusCode::OK, response_headers, body_stream)) + }; + match build_static_response_f.await { + Ok((status, headers, body_stream)) => { + return (status, headers, body_stream).into_response() + } + Err(e) => return e.into_response(), + } + } + let headers = headers .iter() .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) diff --git a/backend/windmill-api/src/job_helpers_ee.rs b/backend/windmill-api/src/job_helpers_ee.rs index 61a946bb84..b64ccde92d 100644 --- a/backend/windmill-api/src/job_helpers_ee.rs +++ b/backend/windmill-api/src/job_helpers_ee.rs @@ -1,5 +1,23 @@ use axum::Router; +#[cfg(feature = "parquet")] +use crate::db::{ApiAuthed, DB}; +#[cfg(feature = "parquet")] +use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource}; + pub fn workspaced_service() -> Router { Router::new() } + +#[cfg(feature = "parquet")] +pub async fn get_workspace_s3_resource<'c>( + _authed: &ApiAuthed, + _db: &DB, + _user_db: Option, + _token: &str, + _w_id: &str, + _storage: Option, +) -> windmill_common::error::Result<(Option, Option)> { + // implementation is not open source + Ok((None, None)) +} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 4547c29d51..715615f37f 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -109,10 +109,13 @@ pub struct S3AwsOidcResource { pub audience: Option, } -#[derive(Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone)] pub struct S3Object { pub s3: String, + #[serde(skip_serializing_if = "Option::is_none")] pub storage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub filename: Option, } #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 80269f7777..b406226120 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -799,6 +799,7 @@ pub async fn get_cached_resource_value_if_valid( S3Object { s3: s3_file_key.clone(), storage: cached_resource.storage.clone(), + filename: None, }, ) .await; diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index bd4270a836..57adc334b4 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -299,7 +299,7 @@ let storage: string | undefined = undefined export async function open( - preSelectedFileKey: { s3: string; storage: string | undefined } | undefined = undefined + preSelectedFileKey: { s3: string; storage?: string } | undefined = undefined ) { storage = preSelectedFileKey?.storage if (preSelectedFileKey !== undefined) { diff --git a/frontend/src/lib/components/common/fileInput/FileInput.svelte b/frontend/src/lib/components/common/fileInput/FileInput.svelte index 1ce7347143..b123fbb3a4 100644 --- a/frontend/src/lib/components/common/fileInput/FileInput.svelte +++ b/frontend/src/lib/components/common/fileInput/FileInput.svelte @@ -4,6 +4,7 @@ import Button from '../../common/button/Button.svelte' import { twMerge } from 'tailwind-merge' import type { ReadFileAs } from './model' + import { sendUserToast } from '$lib/toast' type ConvertedFile = string | ArrayBuffer | null @@ -89,7 +90,12 @@ event.preventDefault() if (event.dataTransfer) { if (event.dataTransfer.files && event.dataTransfer.files.length) { - onChange(event.dataTransfer.files) + if (!multiple && event.dataTransfer.files.length > 1) { + sendUserToast('Only one file can be uploaded at a time') + return + } else { + onChange(event.dataTransfer.files) + } } } } diff --git a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte index 4ba06e3d99..a2e579a86c 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte @@ -14,7 +14,9 @@ export let acceptedFileTypes: string[] | undefined = ['*'] export let allowMultiple: boolean = true - export let containerText: string = 'Drag and drop files here or click to browse' + export let containerText: string = allowMultiple + ? 'Drag and drop files here or click to browse' + : 'Drag and drop a file here or click to browse' export let customResourcePath: string | undefined = undefined export let customResourceType: 's3' | 'azure_blob' | undefined = undefined // when customResourcePath is provided, this should be provided as well. Will default to S3 if not export let customClass: string = '' @@ -109,6 +111,10 @@ if (fileExtension) { params.append('file_extension', fileExtension) } + if (fileToUpload.type) { + params.append('content_type', fileToUpload.type) + } + // let response = await fetch( // `/api/w/${$workspaceStore}/job_helpers/multipart_upload_s3_file?${params.toString()}`, // { @@ -400,7 +406,6 @@ accept={acceptedFileTypes?.join(',')} multiple={allowMultiple} returnFileNames - includeMimeType on:change={({ detail }) => { forceDisplayUploads = false handleChange(detail) diff --git a/frontend/src/lib/components/triggers/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/RouteEditorInner.svelte index 5cf683515b..10eaef0725 100644 --- a/frontend/src/lib/components/triggers/RouteEditorInner.svelte +++ b/frontend/src/lib/components/triggers/RouteEditorInner.svelte @@ -10,13 +10,18 @@ import { canWrite, emptyString, sendUserToast } from '$lib/utils' import { createEventDispatcher } from 'svelte' import Section from '$lib/components/Section.svelte' - import { Loader2, Save } from 'lucide-svelte' + import { Loader2, Save, Pipette } from 'lucide-svelte' import Label from '$lib/components/Label.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import { page } from '$app/stores' import { isCloudHosted } from '$lib/cloud' import { base } from '$lib/base' + import S3FilePicker from '../S3FilePicker.svelte' + import Toggle from '../Toggle.svelte' + import JsonEditor from '../apps/editor/settingsPanel/inputEditor/JsonEditor.svelte' + import FileUpload from '../common/fileUpload/FileUpload.svelte' + import SimpleEditor from '../SimpleEditor.svelte' let is_flow: boolean = false let initialPath = '' @@ -64,6 +69,8 @@ initialScriptPath = '' fixedScriptPath = fixedScriptPath_ ?? '' script_path = fixedScriptPath + static_asset_config = undefined + s3FileUploadRawMode = false path = '' initialPath = '' dirtyPath = false @@ -81,6 +88,12 @@ let initialRoutePath = '' let route_path = '' let http_method: 'get' | 'post' | 'put' | 'patch' | 'delete' = 'post' + let static_asset_config: { s3: string; storage?: string; filename?: string } | undefined = + undefined + + let s3FilePicker: S3FilePicker + let s3FileUploadRawMode = false + let s3Editor: SimpleEditor | undefined = undefined const dispatch = createEventDispatcher() @@ -100,6 +113,8 @@ http_method = s.http_method ?? 'post' is_async = s.is_async requires_auth = s.requires_auth + static_asset_config = s.static_asset_config + s3FileUploadRawMode = !!static_asset_config can_write = canWrite(s.path, s.extra_perms, $userStore) } @@ -116,7 +131,8 @@ is_async, requires_auth, route_path: $userStore?.is_admin || $userStore?.is_super_admin ? route_path : undefined, - http_method + http_method, + static_asset_config } }) sendUserToast(`Route ${path} updated`) @@ -130,7 +146,8 @@ is_async, requires_auth, route_path, - http_method + http_method, + static_asset_config } }) sendUserToast(`Route ${path} created`) @@ -181,6 +198,17 @@ $: validateRoute(route_path, http_method) +{#if static_asset_config} + { + s3Editor?.setCode(JSON.stringify(static_asset_config, null, 2)) + }} + readOnlyMode={false} + /> +{/if} + Save @@ -286,35 +318,107 @@ -
-

- Pick a script or flow to be triggered
- To handle headers, query or path parameters, add a preprocessor to your runnable. -

-
- +
+ { + if (ev.detail === 'static_asset') { + static_asset_config = { s3: '' } + script_path = '' + initialScriptPath = '' + is_flow = false + } else { + static_asset_config = undefined + } + }} + > + + + - {#if script_path === undefined} - - {/if} -
+ options={{ left: 'Existing file' }} + /> + {#if s3FileUploadRawMode} + { + dispatch('focus') + }} + on:blur={(e) => { + dispatch('blur') + }} + code={JSON.stringify(static_asset_config ?? { s3: '' }, null, 2)} + bind:value={static_asset_config} + /> + + {:else} + { + static_asset_config = { + s3: evt.detail?.path ?? '', + filename: evt.detail?.filename ?? undefined + } + s3FileUploadRawMode = true + }} + on:deletion={(evt) => { + static_asset_config = { + s3: '' + } + }} + /> + {/if} + + {:else} +

+ Pick a script or flow to be triggered
+ To handle headers, query or path parameters, add a preprocessor to your runnable. +

+
+ + + {#if script_path === undefined} + + {/if} +
+ {/if}
+
diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index bfa2760195..6818749627 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -358,15 +358,15 @@ class Windmill: "percent": value, "flow_job_id": flow_id or None, }, - ) + ) - def get_progress(self, job_id: Optional[str] = None ) -> Any: + def get_progress(self, job_id: Optional[str] = None) -> Any: workspace = get_workspace() job_id = job_id or os.environ.get("WM_JOB_ID") r = self.get( f"/w/{workspace}/job_metrics/get_progress/{job_id}", - ) + ) if r.status_code == 404: print(f"Job {job_id} does not exist") return None @@ -472,7 +472,13 @@ class Windmill: print(file_reader.read()) ''' """ - reader = S3BufferedReader(f"{self.workspace}", self.client, s3object["s3"], s3_resource_path, s3object["storage"] if "storage" in s3object else None) + reader = S3BufferedReader( + f"{self.workspace}", + self.client, + s3object["s3"], + s3_resource_path, + s3object["storage"] if "storage" in s3object else None, + ) return reader def write_s3_file( @@ -480,6 +486,8 @@ class Windmill: s3object: S3Object | None, file_content: BufferedReader | bytes, s3_resource_path: str | None, + content_type: str | None, + content_disposition: str | None, ) -> S3Object: """ Write a file to the workspace S3 bucket @@ -511,8 +519,12 @@ class Windmill: query_params["file_key"] = s3object["s3"] if s3_resource_path is not None and s3_resource_path != "": query_params["s3_resource_path"] = s3_resource_path - if s3object is not None and "storage" in s3object and s3object["storage"] is not None: + if s3object is not None and "storage" in s3object and s3object["storage"] is not None: query_params["storage"] = s3object["storage"] + if content_type is not None: + query_params["content_type"] = content_type + if content_disposition is not None: + query_params["content_disposition"] = content_disposition try: # need a vanilla client b/c content-type is not application/json here @@ -865,6 +877,7 @@ def set_state(value: Any) -> None: """ return _client.set_state(value) + @init_global_client def set_progress(value: int, job_id: Optional[str] = None) -> None: """ @@ -872,6 +885,7 @@ def set_progress(value: int, job_id: Optional[str] = None) -> None: """ return _client.set_progress(value, job_id) + @init_global_client def get_progress(job_id: Optional[str] = None) -> Any: """