diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index ac611acf82..bfd8dd5939 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -3,11 +3,11 @@ use rustpython_parser::{ast::Suite, Parse}; use std::collections::HashMap; use windmill_parser::asset_parser::{ asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsResult, + ParseAssetsOutput, ParseAssetsResult, }; use AssetUsageAccessType::*; -pub fn parse_assets(input: &str) -> anyhow::Result> { +pub fn parse_assets(input: &str) -> anyhow::Result { let ast = Suite::parse(input, "main.py") .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; @@ -25,7 +25,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result> { } } - Ok(merge_assets(assets_finder.assets)) + Ok(ParseAssetsOutput { assets: merge_assets(assets_finder.assets), ..Default::default() }) } type VarAssetName = String; @@ -216,7 +216,7 @@ impl AssetsFinder { match windmill_parser_sql::parse_assets(&sql) { Ok(mut sql_assets) => { if let Some(schema_name) = schema { - for asset in &mut sql_assets { + for asset in &mut sql_assets.assets { if asset.kind == *kind && asset.path.starts_with(path.as_str()) { asset.path = format!( "{}/{}.{}", @@ -227,7 +227,7 @@ impl AssetsFinder { } } } - self.assets.extend(sql_assets); + self.assets.extend(sql_assets.assets); } _ => {} } @@ -290,7 +290,7 @@ import wmill def main(): wmill.load_s3_file('s3:///test.csv') "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -308,7 +308,7 @@ import wmill def main(): db = wmill.datatable() "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -327,7 +327,7 @@ def main(x: int): db = wmill.datatable('dt') return db.query('SELECT * FROM friends WHERE age = $1', x).fetch() "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -348,7 +348,7 @@ def main(x: int): db.query('SELECT * FROM friends WHERE age = $1', x).fetch_one() db.query('SELECT * FROM analytics').fetch() "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ @@ -380,7 +380,7 @@ def main(): def g(): db = wmill.ducklake('another2') "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ @@ -412,7 +412,7 @@ def main(): def g(): db = wmill.ducklake() "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ @@ -438,7 +438,7 @@ def main(x: int): db = wmill.datatable('dt:public') return db.query('SELECT * FROM friends WHERE age = $1', x).fetch() "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -457,7 +457,7 @@ def main(): db = wmill.ducklake('lake1:analytics') return db.query('SELECT * FROM metrics').fetch() "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -477,7 +477,7 @@ def main(x: int): db.query('INSERT INTO users VALUES ($1)', x).fetch() return db.query('SELECT * FROM users').fetch() "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -495,7 +495,7 @@ import wmill def main(): db = wmill.datatable('dt:public') "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|o| o.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser.rs b/backend/parsers/windmill-parser-sql/src/asset_parser.rs index 818a06f0ca..489f9c661f 100644 --- a/backend/parsers/windmill-parser-sql/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql/src/asset_parser.rs @@ -10,11 +10,11 @@ use sqlparser::{ }; use windmill_parser::asset_parser::{ asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsResult, + ParseAssetsOutput, ParseAssetsResult, }; use AssetUsageAccessType::*; -pub fn parse_assets(input: &str) -> anyhow::Result> { +pub fn parse_assets(input: &str) -> anyhow::Result { let statements = Parser::parse_sql(&DuckDbDialect, input)?; let mut collector = AssetCollector::new(); @@ -30,7 +30,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result> { } } - Ok(merge_assets(collector.assets)) + Ok(ParseAssetsOutput { assets: merge_assets(collector.assets), ..Default::default() }) } /// Visitor that collects S3 asset literals from SQL statements @@ -402,7 +402,7 @@ mod tests { SELECT * FROM read_parquet('s3:///a.parquet'); COPY (SELECT * FROM 's3://snd/b.parquet') TO 's3:///c.parquet'; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ @@ -432,7 +432,7 @@ mod tests { SELECT 2; USE dl; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -449,7 +449,7 @@ mod tests { ATTACH 'ducklake://my_dl' AS dl; SELECT * FROM dl.table1; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -467,7 +467,7 @@ mod tests { SELECT dt.read_bait FROM unrelated_table; -- dt. doesn't access the asset INSERT INTO dt.table1 VALUES ('test'); "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -485,7 +485,7 @@ mod tests { DETACH dl; SELECT * FROM dl.table1; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![])); } @@ -498,7 +498,7 @@ mod tests { USE memory; SELECT * FROM table1; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -515,7 +515,7 @@ mod tests { ATTACH 'datatable' AS dl; INSERT INTO dl.table1 VALUES ('test'); "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -537,7 +537,7 @@ mod tests { INSERT INTO friends VALUES ($name, $age); SELECT * FROM friends; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -555,7 +555,7 @@ mod tests { ATTACH 'ducklake' AS dl; USE dl; SELECT * FROM a_function(''); "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -573,7 +573,7 @@ mod tests { USE dl; DELETE FROM table1; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -591,7 +591,7 @@ mod tests { USE dl; UPDATE table1 SET id = NULL; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -609,7 +609,7 @@ mod tests { USE db; SELECT * FROM table1; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -626,7 +626,7 @@ mod tests { ATTACH 'ducklake' AS dl; UPDATE dl.table1 SET id = NULL; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -644,7 +644,7 @@ mod tests { UPDATE dl.sch.table1 SET id = NULL; SELECT * FROM dl.sch.table1; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { @@ -663,7 +663,7 @@ mod tests { UPDATE sch.table1 SET id = NULL; SELECT * FROM sch.table1; "#; - let s = parse_assets(input); + let s = parse_assets(input).map(|s| s.assets); assert_eq!( s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index 426fd8f8c0..787c69c2e4 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -1,16 +1,16 @@ use std::collections::HashMap; -use swc_common::{sync::Lrc, FileName, SourceMap}; +use swc_common::{sync::Lrc, FileName, SourceMap, Spanned}; use swc_ecma_ast::{CallExpr, Expr, Lit, MemberExpr, MemberProp, Str}; use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; use swc_ecma_visit::{Visit, VisitWith}; use windmill_parser::asset_parser::{ asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsResult, + ParseAssetsOutput, ParseAssetsResult, SqlQueryDetails, }; use AssetUsageAccessType::*; -pub fn parse_assets(code: &str) -> anyhow::Result> { +pub fn parse_assets(code: &str) -> anyhow::Result { let cm: Lrc = Default::default(); let fm = cm.new_source_file(FileName::Custom("main.ts".into()).into(), code.into()); let lexer = Lexer::new( @@ -35,9 +35,13 @@ pub fn parse_assets(code: &str) -> anyhow::Result> { anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {err_s}, {e:?}") })? .body; - let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() }; + let mut assets_finder = + AssetsFinder { assets: vec![], sql_queries: vec![], var_identifiers: HashMap::new() }; assets_finder.visit_module_items(&ast); - Ok(merge_assets(assets_finder.assets)) + Ok(ParseAssetsOutput { + assets: merge_assets(assets_finder.assets), + sql_queries: assets_finder.sql_queries, + }) } type VarAssetName = String; @@ -52,6 +56,8 @@ struct AssetsFinder { // so that when we see a tagged template expression with tag "sql" we know which datatable it // corresponds to. This allows us to infer if a datatable is Read or Write based on the SQL query. var_identifiers: HashMap, + + sql_queries: Vec, } /// Helper function to extract wmill.datatable() or wmill.ducklake() calls, @@ -220,26 +226,47 @@ impl Visit for AssetsFinder { }; // Extract the SQL query from the template quasis (string parts) + // Substitute ${} with $1, $2, etc. let sql: String = node .tpl .quasis .iter() .map(|quasi| quasi.raw.as_str()) - .collect::>() - .join("$1"); // placeholder for expressions + .enumerate() + .fold(String::new(), |acc, (i, s)| { + if i == 0 { + s.to_string() + } else { + format!("{}${}{}", acc, i, s) + } + }); let duckdb_conn_prefix = match kind { AssetKind::DataTable => "datatable", AssetKind::Ducklake => "ducklake", _ => return, }; - let sql = format!("ATTACH '{duckdb_conn_prefix}://{asset_name}' AS dt; USE dt; {sql}"); + + // Capture SQL query details before transforming for SQL parser + let span = node.span(); + let span_tuple = (span.lo.0, span.hi.0); + + self.sql_queries.push(SqlQueryDetails { + query_string: sql.clone(), + span: span_tuple, + source_kind: *kind, + source_name: asset_name.clone(), + source_schema: schema.clone(), + }); + + let sql_with_attach = + format!("ATTACH '{duckdb_conn_prefix}://{asset_name}' AS dt; USE dt; {sql}"); // We use the SQL parser to detect if it's a read or write query - match windmill_parser_sql::parse_assets(&sql) { + match windmill_parser_sql::parse_assets(&sql_with_attach) { Ok(mut sql_assets) => { if let Some(schema) = schema { - for asset in &mut sql_assets { + for asset in &mut sql_assets.assets { if asset.kind == *kind && asset.path.starts_with(asset_name) { asset.path = format!( "{}/{}.{}", @@ -250,7 +277,7 @@ impl Visit for AssetsFinder { } } } - self.assets.extend(sql_assets); + self.assets.extend(sql_assets.assets); } _ => {} } @@ -307,7 +334,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, path: "/test.csv".to_string(), @@ -326,7 +353,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt".to_string(), @@ -346,7 +373,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/friends".to_string(), @@ -368,7 +395,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ ParseAssetsResult { kind: AssetKind::DataTable, @@ -403,7 +430,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ ParseAssetsResult { kind: AssetKind::DataTable, @@ -437,7 +464,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ ParseAssetsResult { kind: AssetKind::DataTable, @@ -464,7 +491,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "main/myschema.friends".to_string(), @@ -485,7 +512,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/public.users".to_string(), @@ -504,7 +531,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt".to_string(), @@ -525,7 +552,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::DataTable, path: "dt/users".to_string(), @@ -547,7 +574,7 @@ mod tests { "#; let s = parse_assets(input); assert_eq!( - s.map_err(|e| e.to_string()), + s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ ParseAssetsResult { kind: AssetKind::DataTable, @@ -562,4 +589,95 @@ mod tests { ]) ); } + + #[test] + fn test_ts_asset_parser_sql_query_details() { + let input = r#" + import * as wmill from "windmill-client" + export async function main(x: number) { + let sql = wmill.datatable('dt') + return await sql`SELECT * FROM friends WHERE age = ${x}`.fetch() + } + "#; + let result = parse_assets(input).unwrap(); + + // Check assets + assert_eq!(result.assets.len(), 1); + assert_eq!(result.assets[0].kind, AssetKind::DataTable); + assert_eq!(result.assets[0].path, "dt/friends"); + + // Check SQL query details + assert_eq!(result.sql_queries.len(), 1); + let query_detail = &result.sql_queries[0]; + assert_eq!( + query_detail.query_string, + "SELECT * FROM friends WHERE age = $1" + ); + assert_eq!(query_detail.source_kind, AssetKind::DataTable); + assert_eq!(query_detail.source_name, "dt"); + assert_eq!(query_detail.source_schema, None); + // Span should be non-zero + assert!(query_detail.span.0 > 0); + assert!(query_detail.span.1 > query_detail.span.0); + } + + #[test] + fn test_ts_asset_parser_sql_query_details_with_schema() { + let input = r#" + import * as wmill from "windmill-client" + export async function main(x: number) { + let sql = wmill.datatable('dt:public') + await sql`INSERT INTO users VALUES (${x})`.fetch() + return await sql`SELECT * FROM users`.fetch() + } + "#; + let result = parse_assets(input).unwrap(); + + // Check SQL query details + assert_eq!(result.sql_queries.len(), 2); + + // First query (INSERT) + assert_eq!( + result.sql_queries[0].query_string, + "INSERT INTO users VALUES ($1)" + ); + assert_eq!(result.sql_queries[0].source_kind, AssetKind::DataTable); + assert_eq!(result.sql_queries[0].source_name, "dt"); + assert_eq!( + result.sql_queries[0].source_schema, + Some("public".to_string()) + ); + + // Second query (SELECT) + assert_eq!(result.sql_queries[1].query_string, "SELECT * FROM users"); + assert_eq!(result.sql_queries[1].source_kind, AssetKind::DataTable); + assert_eq!(result.sql_queries[1].source_name, "dt"); + assert_eq!( + result.sql_queries[1].source_schema, + Some("public".to_string()) + ); + } + + #[test] + fn test_ts_asset_parser_sql_query_details_ducklake() { + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + let sql = wmill.ducklake('my_lake') + return await sql`SELECT id, name FROM products LIMIT 10`.fetch() + } + "#; + let result = parse_assets(input).unwrap(); + + // Check SQL query details + assert_eq!(result.sql_queries.len(), 1); + let query_detail = &result.sql_queries[0]; + assert_eq!( + query_detail.query_string, + "SELECT id, name FROM products LIMIT 10" + ); + assert_eq!(query_detail.source_kind, AssetKind::Ducklake); + assert_eq!(query_detail.source_name, "my_lake"); + assert_eq!(query_detail.source_schema, None); + } } diff --git a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs index 48f7984334..9e4cedf08d 100644 --- a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs @@ -1,10 +1,10 @@ use windmill_parser::asset_parser::{ - merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult, + merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, }; use crate::{parse_ansible_reqs, ResourceOrVariablePath}; -pub fn parse_assets(input: &str) -> anyhow::Result> { +pub fn parse_assets(input: &str) -> anyhow::Result { let mut assets = vec![]; if let (_, Some(ansible_reqs), _) = parse_ansible_reqs(input)? { if let Some(delegate_to_git_repo_details) = ansible_reqs.delegate_to_git_repo { @@ -36,5 +36,5 @@ pub fn parse_assets(input: &str) -> anyhow::Result> { } } - Ok(merge_assets(assets)) + Ok(ParseAssetsOutput { assets: merge_assets(assets), ..Default::default() }) } diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 5e3af4f831..7c5a6ec437 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -958,7 +958,7 @@ dependencies: content: "{{ my_result | to_json }}" dest: result.json "#; - let a = parse_assets(p).unwrap(); + let a = parse_assets(p).unwrap().assets; println!("The resulting assets are: {}", a.len()); let a = parse_ansible_reqs(p).unwrap(); diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 962fa627b4..0d595ae07d 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -27,6 +27,22 @@ pub struct ParseAssetsResult { pub access_type: Option, // None in case of ambiguity } +#[derive(Serialize, Debug, PartialEq)] +pub struct SqlQueryDetails { + pub query_string: String, // SQL query with $1 placeholders for interpolations + pub span: (u32, u32), // (start, end) byte positions in source code + pub source_kind: AssetKind, // DataTable or Ducklake + pub source_name: String, // e.g., "main", "dt" + #[serde(skip_serializing_if = "Option::is_none")] + pub source_schema: Option, // e.g., Some("public"), None +} + +#[derive(Serialize, Debug, Default)] +pub struct ParseAssetsOutput { + pub assets: Vec, + pub sql_queries: Vec, +} + #[derive(Debug, Clone, Serialize)] pub struct DelegateToGitRepoDetails { pub resource: String, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 557d8cc790..f1ad0697ad 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3029,6 +3029,7 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/edit_git_sync_config: post: summary: edit workspace git sync settings diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 15b9c5ec95..82d7911847 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -330,6 +330,21 @@ async fn reset() -> () { todo!() } +#[derive(Serialize, Debug)] +pub struct PrepareQueryColumnInfo { + pub name: String, + #[serde(rename = "type")] + pub type_name: String, +} + +#[derive(Serialize, Debug)] +pub struct PrepareQueryResult { + #[serde(skip_serializing_if = "Option::is_none")] + pub columns: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + #[derive(Deserialize, Serialize)] pub struct PgDatabase { pub host: String, @@ -460,7 +475,7 @@ impl PgDatabase { let password = parsed_url.password().map(|p| p.to_string()); let password = match password { Some(p) => Some(urlencoding::decode(&p).map_err(to_anyhow)?.to_string()), - None => None, + None => None, }; let host = parsed_url .host_str() diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index d11fc3a40b..3597e75f50 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -707,6 +707,7 @@ pub struct TypeScriptAnnotations { pub struct SqlAnnotations { pub return_last_result: bool, // deprecated, use result_collection instead pub result_collection: SqlResultCollectionStrategy, + pub prepare: bool, // Used to prepare datatable queries without executing } #[annotations("#")] diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 68b75f7fce..7e13464dde 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -27,6 +27,7 @@ use crate::handle_child::run_future_with_polling_update_job_poller; #[cfg(feature = "mysql")] use crate::mysql_executor::MysqlDatabase; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; +use crate::sql_utils::remove_comments; use windmill_common::client::AuthedClient; use windmill_common::s3_helpers::DEFAULT_STORAGE; @@ -685,71 +686,10 @@ pub struct Arg { pub json_value: serde_json::Value, } -// input should contain a single statement. remove all comments before and after it -fn remove_comments(stmt: &str) -> &str { - let mut in_stmt = false; - let mut in_comment = false; - let mut start = None; - let mut end = stmt.len(); - - let mut c = ' '; - for (next_i, next_char) in stmt.char_indices() { - if next_i > 0 { - let i = next_i - 1; - if !in_comment && in_stmt && c == ';' { - end = i + 1; - break; - } else if in_comment && c == '\n' { - in_comment = false; - } else if c == '-' && next_char == '-' { - in_comment = true; - } else if !in_comment && !c.is_whitespace() && start == None { - start = Some(i); - in_stmt = true; - } - } - c = next_char; - } - - return &stmt[start.unwrap_or(0)..end]; -} - #[cfg(test)] mod tests { use super::*; - // Tests for remove_comments function - #[test] - fn test_remove_comments_single_line() { - let sql = "-- This is a comment\nSELECT * FROM table;"; - assert_eq!(remove_comments(sql), "SELECT * FROM table;"); - } - #[test] - fn test_remove_comments_multi_line() { - let sql = "-- This is a comment\nSELECT * FROM table;\n-- Another comment"; - assert_eq!(remove_comments(sql), "SELECT * FROM table;"); - } - #[test] - fn test_remove_comments_inline_comment() { - let sql = " SELECT * FROM table; -- This is an inline comment "; - assert_eq!(remove_comments(sql), "SELECT * FROM table;"); - } - #[test] - fn test_remove_comments_no_comments() { - let sql = "SELECT * FROM table;"; - assert_eq!(remove_comments(sql), "SELECT * FROM table;"); - } - #[test] - fn test_remove_comments_empty_string() { - let sql = ""; - assert_eq!(remove_comments(sql), ""); - } - #[test] - fn test_remove_comments_with_whitespace() { - let sql = " -- Comment\n -- Comment2\n -- Comment3\n SELECT\n\n * FROM\n table\n;\n\n -- end comment "; - assert_eq!(remove_comments(sql), "SELECT\n\n * FROM\n table\n;"); - } - // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { @@ -1107,19 +1047,4 @@ mod tests { let serialized = serde_json::to_string(&arg).unwrap(); assert!(serialized.contains("\"json_value\":{\"key\":\"value\"}")); } - - #[test] - fn test_remove_comments_comment_in_string() { - let sql = "SELECT '-- not a comment' FROM table;"; - let result = remove_comments(sql); - assert_eq!(result, "SELECT '-- not a comment' FROM table;"); - } - - #[test] - fn test_remove_comments_multiple_dashes() { - let sql = "SELECT 5 - - 3;"; - let result = remove_comments(sql); - // This correctly handles the subtraction of negative number - assert_eq!(result, sql); - } } diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 6c9663d2d2..d211695091 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -65,6 +65,7 @@ mod rust_executor; mod sanitized_sql_params; mod schema; pub mod scoped_dependency_map; +pub mod sql_utils; mod universal_pkg_installer; mod worker; mod worker_flow; diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index befb3bb579..3540cd0bdd 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -29,7 +29,7 @@ use windmill_common::worker::{ to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED, }; use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; -use windmill_common::PgDatabase; +use windmill_common::{PgDatabase, PrepareQueryColumnInfo, PrepareQueryResult}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_s3_mode, @@ -44,6 +44,7 @@ use crate::common::{ }; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; +use crate::sql_utils::remove_comments; use crate::MAX_RESULT_SIZE; use bytes::Buf; use lazy_static::lazy_static; @@ -299,6 +300,34 @@ pub async fn do_postgresql( let result_f = async move { let mut results = vec![]; for (i, query) in queries.iter().enumerate() { + if annotations.prepare { + let query = remove_comments(query); + // Used by the data table typechecker to set default schemas + if query.starts_with("SET search_path") || query.starts_with("RESET search_path") { + let _ = client.execute(&query.to_string(), &[]).await; + continue; + } + let prepared = client.prepare(&query).await; + let prepared = match prepared { + Ok(prepared) => { + let columns: Option> = Some( + prepared + .columns() + .iter() + .map(|col| PrepareQueryColumnInfo { + name: col.name().to_string(), + type_name: col.type_().name().to_string(), + }) + .collect(), + ); + PrepareQueryResult { columns, error: None } + } + Err(e) => PrepareQueryResult { columns: None, error: Some(e.to_string()) }, + }; + results.push(vec![to_raw_value(&prepared)]); + continue; + } + let result = do_postgresql_inner( query.to_string(), ¶m_idx_to_arg_and_value, diff --git a/backend/windmill-worker/src/sql_utils.rs b/backend/windmill-worker/src/sql_utils.rs new file mode 100644 index 0000000000..6784b7c930 --- /dev/null +++ b/backend/windmill-worker/src/sql_utils.rs @@ -0,0 +1,122 @@ +// input should contain a single statement. remove all comments before and after it +pub fn remove_comments(stmt: &str) -> &str { + let mut in_stmt = false; + let mut in_line_comment = false; + let mut in_block_comment = false; + let mut in_string = false; + let mut string_delimiter = '\0'; + let mut start = None; + let mut end = stmt.len(); + + let chars: Vec = stmt.chars().collect(); + let len = chars.len(); + + for i in 0..len { + let c = chars[i]; + let next_char = if i + 1 < len { chars[i + 1] } else { '\0' }; + let prev_char = if i > 0 { chars[i - 1] } else { '\0' }; + + // Handle string literals (single or double quotes) + if !in_line_comment && !in_block_comment { + if (c == '\'' || c == '"') && prev_char != '\\' { + if in_string && c == string_delimiter { + // Exiting string + in_string = false; + string_delimiter = '\0'; + } else if !in_string { + // Entering string + in_string = true; + string_delimiter = c; + } + } + } + + // Handle comments only when not inside a string + if !in_string { + // Check for start of line comment + if !in_block_comment && c == '-' && next_char == '-' { + in_line_comment = true; + } + // Check for end of line comment + else if in_line_comment && c == '\n' { + in_line_comment = false; + } + // Check for start of block comment + else if !in_line_comment && c == '/' && next_char == '*' { + in_block_comment = true; + } + // Check for end of block comment + else if in_block_comment && c == '*' && next_char == '/' { + in_block_comment = false; + // Skip the closing '/' by continuing after incrementing i in the loop + continue; + } + } + + // Track statement boundaries + if !in_line_comment && !in_block_comment && !in_string { + // Mark start of statement + if !in_stmt && !c.is_whitespace() { + start = Some(i); + in_stmt = true; + } + // Mark end of statement at semicolon + if in_stmt && c == ';' { + end = i + 1; + break; + } + } + } + + &stmt[start.unwrap_or(0)..end] +} + +#[cfg(test)] +mod tests { + use super::*; + + // Tests for remove_comments function + #[test] + fn test_remove_comments_single_line() { + let sql = "-- This is a comment\nSELECT * FROM table;"; + assert_eq!(remove_comments(sql), "SELECT * FROM table;"); + } + #[test] + fn test_remove_comments_multi_line() { + let sql = "-- This is a comment\nSELECT * FROM table;\n-- Another comment"; + assert_eq!(remove_comments(sql), "SELECT * FROM table;"); + } + #[test] + fn test_remove_comments_inline_comment() { + let sql = " SELECT * FROM table; -- This is an inline comment "; + assert_eq!(remove_comments(sql), "SELECT * FROM table;"); + } + #[test] + fn test_remove_comments_no_comments() { + let sql = "SELECT * FROM table;"; + assert_eq!(remove_comments(sql), "SELECT * FROM table;"); + } + #[test] + fn test_remove_comments_empty_string() { + let sql = ""; + assert_eq!(remove_comments(sql), ""); + } + #[test] + fn test_remove_comments_with_whitespace() { + let sql = " -- Comment\n -- Comment2\n -- Comment3\n SELECT\n\n * FROM\n table\n;\n\n -- end comment "; + assert_eq!(remove_comments(sql), "SELECT\n\n * FROM\n table\n;"); + } + #[test] + fn test_remove_comments_comment_in_string() { + let sql = "SELECT '-- not a comment' FROM table;"; + let result = remove_comments(sql); + assert_eq!(result, "SELECT '-- not a comment' FROM table;"); + } + #[test] + fn test_remove_comments_multiple_dashes() { + let sql = "SELECT 5 - - 3;"; + let result = remove_comments(sql); + // This correctly handles the subtraction of negative number + assert_eq!(result, sql); + } +} diff --git a/cli/src/guidance/prompts.ts b/cli/src/guidance/prompts.ts index bde6ce11b9..c745c205d3 100644 --- a/cli/src/guidance/prompts.ts +++ b/cli/src/guidance/prompts.ts @@ -1561,7 +1561,7 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord * WHERE name = \${name} AND age = \${age}::int * \`.fetch() */ -datatable(name: string = "main"): SqlTemplateFunction +datatable(name: string = "main"): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries @@ -2621,4 +2621,3 @@ workspace related commands - \`-y --yes\` - Skip confirmation prompt `; - diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b9ff10d75d..87110e0c2f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -79,12 +79,12 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.594.0", - "windmill-parser-wasm-regex": "1.592.1", + "windmill-parser-wasm-py": "1.595.0", + "windmill-parser-wasm-regex": "1.593.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", - "windmill-parser-wasm-ts": "1.592.1", - "windmill-parser-wasm-yaml": "1.561.0", + "windmill-parser-wasm-ts": "1.593.0", + "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", @@ -261,7 +261,6 @@ "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", @@ -277,7 +276,6 @@ "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -1859,7 +1857,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -2369,6 +2366,7 @@ "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", @@ -2914,6 +2912,7 @@ "integrity": "sha512-7TSvMrCdmig5TMyYDW876C5FljhA0wlGixtvASCiqUqtLfmyEEpaysXjC7GhR5mWcGRrCGF+L2Bl1eEaW1wTCA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "@sveltejs/acorn-typescript": "^1.0.5", @@ -2991,6 +2990,7 @@ "integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", "debug": "^4.4.1", @@ -3497,8 +3497,7 @@ "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/ms": { "version": "2.1.0", @@ -3511,8 +3510,7 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/semver": { "version": "7.7.1", @@ -3582,6 +3580,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -3772,6 +3771,7 @@ "integrity": "sha512-94yVpDbb+ykiT7mK6ToonGnq2GIHEQGBTZTAzGxBGQXcVNCh54YKC2/WkfaDzxy0m6Kgw05kq3FYHKHu+wRdIA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/browser": "4.0.15", "@vitest/mocker": "4.0.15", @@ -4021,6 +4021,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4074,6 +4075,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4219,7 +4221,6 @@ "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -4247,7 +4248,6 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -4340,8 +4340,7 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz", "integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/base64-js": { "version": "1.5.1", @@ -4468,6 +4467,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.9", "caniuse-lite": "^1.0.30001746", @@ -4665,7 +4665,6 @@ "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "camelcase": "^6.3.0", "map-obj": "^4.1.0", @@ -4685,7 +4684,6 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4699,7 +4697,6 @@ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4713,7 +4710,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -4822,6 +4818,7 @@ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", + "peer": true, "dependencies": { "@kurkle/color": "^0.3.0" }, @@ -5062,7 +5059,6 @@ "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", @@ -5127,7 +5123,6 @@ "integrity": "sha512-IQOkD3hbR5KrN93MtcYuad6YPuTSUhntLHDuLEbFWE+ff2/XSZNdZG+LcbbIW5AXKg/WFIfYItIzVoHngHXZzA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12 || >=16" } @@ -5393,6 +5388,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -5446,6 +5442,7 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.21.0" }, @@ -5486,7 +5483,6 @@ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -5500,7 +5496,6 @@ "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "decamelize": "^1.1.0", "map-obj": "^1.0.0" @@ -5518,7 +5513,6 @@ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5529,7 +5523,6 @@ "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -6015,7 +6008,6 @@ "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -6103,6 +6095,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6649,7 +6642,6 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 4.9.1" } @@ -7135,7 +7127,6 @@ "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "global-prefix": "^3.0.0" }, @@ -7149,7 +7140,6 @@ "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ini": "^1.3.5", "kind-of": "^6.0.2", @@ -7165,7 +7155,6 @@ "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -7215,8 +7204,7 @@ "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz", "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/gopd": { "version": "1.2.0", @@ -7252,6 +7240,7 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz", "integrity": "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw==", "license": "MIT", + "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -7310,7 +7299,6 @@ "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -7532,7 +7520,6 @@ "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -7546,7 +7533,6 @@ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7559,8 +7545,7 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/html-tags": { "version": "3.3.1", @@ -7568,7 +7553,6 @@ "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -7652,7 +7636,6 @@ "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -7673,7 +7656,6 @@ "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7745,8 +7727,7 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -7851,7 +7832,6 @@ "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7862,7 +7842,6 @@ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7961,8 +7940,7 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -7998,8 +7976,7 @@ "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-refs": { "version": "3.0.15", @@ -8112,7 +8089,6 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8691,8 +8667,7 @@ "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.uniq": { "version": "4.5.0", @@ -8760,7 +8735,6 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -8811,7 +8785,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -9052,7 +9025,6 @@ "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/minimist": "^1.2.2", "camelcase-keys": "^7.0.0", @@ -9080,7 +9052,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -9774,7 +9745,6 @@ "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "arrify": "^1.0.1", "is-plain-obj": "^1.1.0", @@ -9853,6 +9823,7 @@ "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-21.6.0.tgz", "integrity": "sha512-YTxKRHe9d4TvyEzWIqLpJXLyZyO4xFlLgrkgHoWBpomm6gIuwaRJJRpapBZf24oG8AhkniZSPg2iv/84M+ho6g==", "license": "MIT", + "peer": true, "dependencies": { "@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": "21.6.0", "@codingame/monaco-vscode-api": "21.6.0" @@ -10104,7 +10075,6 @@ "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -10456,7 +10426,6 @@ "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -10791,6 +10760,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -10979,6 +10949,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "lilconfig": "^3.0.0", "yaml": "^2.3.4" @@ -11368,8 +11339,7 @@ "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz", "integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/postcss-safe-parser": { "version": "6.0.0", @@ -11544,6 +11514,7 @@ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11848,7 +11819,6 @@ "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/normalize-package-data": "^2.4.0", "normalize-package-data": "^3.0.2", @@ -11868,7 +11838,6 @@ "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "find-up": "^5.0.0", "read-pkg": "^6.0.0", @@ -11887,7 +11856,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11901,7 +11869,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11944,7 +11911,6 @@ "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "indent-string": "^5.0.0", "strip-indent": "^4.0.0" @@ -12540,7 +12506,6 @@ "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -12617,7 +12582,6 @@ "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-expression-parse": "^3.0.0", "spdx-license-ids": "^3.0.0" @@ -12628,8 +12592,7 @@ "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", "dev": true, - "license": "CC-BY-3.0", - "peer": true + "license": "CC-BY-3.0" }, "node_modules/spdx-expression-parse": { "version": "3.0.1", @@ -12637,7 +12600,6 @@ "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "spdx-exceptions": "^2.1.0", "spdx-license-ids": "^3.0.0" @@ -12648,8 +12610,7 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz", "integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==", "dev": true, - "license": "CC0-1.0", - "peer": true + "license": "CC0-1.0" }, "node_modules/sprintf-js": { "version": "1.0.3", @@ -12743,7 +12704,6 @@ "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12769,8 +12729,7 @@ "resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz", "integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/style-to-object": { "version": "0.4.4", @@ -12819,7 +12778,6 @@ "integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@csstools/css-parser-algorithms": "^2.3.1", "@csstools/css-tokenizer": "^2.2.0", @@ -12902,7 +12860,6 @@ } ], "license": "MIT-0", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -12916,7 +12873,6 @@ "integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flat-cache": "^3.2.0" }, @@ -12929,8 +12885,7 @@ "resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz", "integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/stylelint/node_modules/postcss-selector-parser": { "version": "6.1.2", @@ -12953,7 +12908,6 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -13088,7 +13042,6 @@ "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -13118,6 +13071,7 @@ "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.39.12.tgz", "integrity": "sha512-CEzwxFuEycokU8K8CE/OuwVbmei+ivu2HvBGYIdASfMa1hCRSNr4RRkzNSvbAvu6h+BOig2CsZTAEY+WKvwZpA==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -13213,21 +13167,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13415,8 +13354,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz", "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", - "dev": true, - "peer": true + "dev": true }, "node_modules/svgo": { "version": "3.3.2", @@ -13467,7 +13405,6 @@ "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", @@ -13495,6 +13432,7 @@ "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -13737,6 +13675,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13799,7 +13738,6 @@ "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13898,6 +13836,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -14112,7 +14051,6 @@ "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-correct": "^3.0.0", "spdx-expression-parse": "^3.0.0" @@ -14167,6 +14105,7 @@ "integrity": "sha512-8wKihlF6EDF8grimwd7GPOhLkQkSIgj6Hlcp0CXhtO3HAXeUUqhgZmJmn07OF8e4PbTusMX6Yxmy1BptVRZsdw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@oxc-project/runtime": "0.99.0", "fdir": "^6.5.0", @@ -14279,6 +14218,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -14312,6 +14252,7 @@ "integrity": "sha512-n1RxDp8UJm6N0IbJLQo+yzLZ2sQCDyl1o0LeugbPWf8+8Fttp29GghsQBjYJVmWq3gBFfe9Hs1spR44vovn2wA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.0.15", "@vitest/mocker": "4.0.15", @@ -14644,14 +14585,14 @@ "integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.594.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.594.0.tgz", - "integrity": "sha512-t34d725JApFzi7GJaJyf3j6ccmRmCAUdlrePQivnHqzjNlG7xvM592Ggr2XMfAIoKqmPG+lhOk+Bs0dPkIPQag==" + "version": "1.595.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.595.0.tgz", + "integrity": "sha512-jkZiEl43J7jyaQDSckMEdSr0hgG3pFVt7pT8MbfNFHV2ywxnkdQsk8fwfdIo3rHHrErPPd4OVRxq0kulCTClXg==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.592.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.592.1.tgz", - "integrity": "sha512-zZzPeYpUUranHgXtXkjBbYGNxys+2p7ESR/NbygYkNSBQZI9g0Lsdri8kQNx8vML9OjryhJbeRUoAZaFimyW4w==" + "version": "1.593.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.593.0.tgz", + "integrity": "sha512-m8BvTGJc2710YODmKKDXiASfssoJ/YFJGfYRhRnQvltENvaRee2NZuf4XoUkOVJESDdCSEX6U6fVkFvF9rXp2Q==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", @@ -14664,14 +14605,14 @@ "integrity": "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.592.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.592.1.tgz", - "integrity": "sha512-jwXVyp718PltzcbRdWksqge9sNDPnzBgaHL8hrCgJMCVfErVHjYk6lv056LNhrgu+hXie4oB0q8A+qs8D9KBdA==" + "version": "1.593.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.593.0.tgz", + "integrity": "sha512-NFY9gaEIpJOwGZJeGYDS3+/16QiPYdxFgmb1bKhXyEAdWjYMQ+otrTezf+K09lmvPzB+len37GlNCU72OgyQ6A==" }, "node_modules/windmill-parser-wasm-yaml": { - "version": "1.561.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.561.0.tgz", - "integrity": "sha512-UbyxsRxJ/QDE+RFjj8q6cMZqr57gxHXBM+W8VLXnQ8I79W5KI+FhKcNFraUpXzqQjalZJ3cVZXXr8C7cTlJ8IQ==" + "version": "1.593.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.593.0.tgz", + "integrity": "sha512-Gyx4aR2jsJYuDrD3mCNTmz7LWOQQXPw5yKNCC1xRgUOPfjsD/tINAFfsBLwVOSmlQQcFZO+wHm4KtDtXOcnGVw==" }, "node_modules/windmill-sql-datatype-parser-wasm": { "version": "1.512.0", @@ -14817,7 +14758,6 @@ "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" @@ -14832,6 +14772,7 @@ "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -15035,7 +14976,6 @@ "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=10" } @@ -15055,6 +14995,7 @@ "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", "license": "MIT", + "peer": true, "dependencies": { "lib0": "^0.2.99" }, @@ -15090,6 +15031,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/frontend/package.json b/frontend/package.json index a3b841b0a0..54572e1276 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -85,12 +85,12 @@ "@anthropic-ai/sdk": "^0.60.0", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-editor-api": "=21.6.0", + "@codingame/monaco-vscode-languages-service-override": "=21.6.0", "@codingame/monaco-vscode-standalone-css-language-features": "=21.6.0", "@codingame/monaco-vscode-standalone-html-language-features": "=21.6.0", "@codingame/monaco-vscode-standalone-json-language-features": "=21.6.0", "@codingame/monaco-vscode-standalone-languages": "=21.6.0", "@codingame/monaco-vscode-standalone-typescript-language-features": "=21.6.0", - "@codingame/monaco-vscode-languages-service-override": "=21.6.0", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", "@redocly/json-to-json-schema": "^0.0.1", @@ -151,12 +151,12 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.594.0", - "windmill-parser-wasm-regex": "1.592.1", + "windmill-parser-wasm-py": "1.595.0", + "windmill-parser-wasm-regex": "1.593.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", - "windmill-parser-wasm-ts": "1.592.1", - "windmill-parser-wasm-yaml": "1.561.0", + "windmill-parser-wasm-ts": "1.593.0", + "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 67549a812b..6756eea4bd 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -78,8 +78,9 @@ SNOWFLAKE_TYPES } from '$lib/consts' import { setupTypeAcquisition, type DepsToGet } from '$lib/ata/index' - import { initWasmTs } from '$lib/infer' + import { initWasmTs, type InferAssetsSqlQueryDetails } from '$lib/infer' import { initVim } from './monaco_keybindings' + import { updateSqlQueriesInWorker, waitForWorkerInitialization } from './sqlTypeService' import { parseTypescriptDeps } from '$lib/relative_imports' import { scriptLangToEditorLang } from '$lib/scripts' @@ -103,7 +104,7 @@ import { getDbSchemas } from './apps/components/display/dbtable/metadata' import { rawAppLintStore, type MonacoLintError } from './raw_apps/lintStore' import { MarkerSeverity } from 'monaco-editor' - import { resource, watch } from 'runed' + import { resource, useDebounce, watch } from 'runed' // import EditorTheme from './EditorTheme.svelte' let divEl: HTMLDivElement | null = $state(null) @@ -137,6 +138,8 @@ enablePreprocessorSnippet?: boolean /** When set, enables raw app lint collection mode and reports Monaco markers to the lint store under this key */ rawAppRunnableKey?: string | undefined + // Used to provide typed queries in TypeScript when detecting assets + preparedAssetsSqlQueries?: InferAssetsSqlQueryDetails[] | undefined } let { @@ -165,7 +168,8 @@ class: clazz = undefined, moduleId = undefined, enablePreprocessorSnippet = false, - rawAppRunnableKey = undefined + rawAppRunnableKey = undefined, + preparedAssetsSqlQueries }: Props = $props() $effect.pre(() => { @@ -1530,9 +1534,9 @@ let disposeTs = languages.typescript.typescriptDefaults.addExtraLib( `export {}; declare module 'windmill-client' { - import { type SqlTemplateFunction } from 'windmill-client'; + import { type DatatableSqlTemplateFunction, type SqlTemplateFunction } from 'windmill-client'; export function ducklake(name${isDucklakeOptional ? '?' : ''}: ${ducklakeNameType}): SqlTemplateFunction; - export function datatable(name${isDataTableOptional ? '?' : ''}: ${datatableNameType}): SqlTemplateFunction; + export function datatable(name${isDataTableOptional ? '?' : ''}: ${datatableNameType}): DatatableSqlTemplateFunction; }`, 'file:///custom_wmill_types.d.ts' ) @@ -1849,6 +1853,46 @@ }) }) + let isTsWorkerInitialized = resource( + [() => lang, () => initialized, () => filePath], + async () => { + if (lang !== 'typescript' || !initialized) return false + console.log('[Editor.isTsWorkerInitialized] Waiting for TS Worker...') + await waitForWorkerInitialization(filePath) + console.log('[Editor.isTsWorkerInitialized] TS Worker initialized successfully') + return true + } + ) + + // Update SQL query type information in the TypeScript worker + // This enables TypeScript to show proper types for SQL template literals + let handleSqlTypingInTs = useDebounce(function handleSqlTypingInTs() { + if (lang !== 'typescript' || !isTsWorkerInitialized.current) return + if (!preparedAssetsSqlQueries || preparedAssetsSqlQueries.length === 0) { + // Clear SQL queries if none exist + updateSqlQueriesInWorker(filePath, []) + return + } + + // Send SQL query information to the custom TypeScript worker + // The worker will inject type parameters into the code that TypeScript analyzes + + // Worker async function call freezes if we pass a Proxy, $state.snapshot() is very important here + updateSqlQueriesInWorker(filePath, $state.snapshot(preparedAssetsSqlQueries)) + }, 250) + + watch( + [ + () => preparedAssetsSqlQueries, + () => lang, + () => filePath, + () => isTsWorkerInitialized.current + ], + () => { + handleSqlTypingInTs() + } + ) + watch([() => customTsTypesData.current], setTypescriptCustomTypes) diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 865e8de4b3..def64ca79d 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -767,6 +767,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS editor?.insertAtBeginning(getImportWmillTsStatement(lang)) } editor?.insertAtCursor(`let sql = wmill.datatable(${name == 'main' ? '' : `'${name}'`})\n`) + editor?.insertAtCursor(`let query_result = await sql\`SELECT * FROM _\`.fetchOne()\n`) } }} tooltip="Attach a datatable to your script." diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index afeea09dd4..8d0d299ab0 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -59,6 +59,8 @@ import JsonInputs from '$lib/components/JsonInputs.svelte' import Toggle from './Toggle.svelte' import { deepEqual } from 'fast-equals' + import { usePreparedAssetSqlQueries } from '$lib/infer.svelte' + import { resource, watch } from 'runed' interface Props { // Exported @@ -148,6 +150,12 @@ shellcheck: false }) + let inferAssetsRes = resource([() => lang, () => code, () => code], () => inferAssets(lang, code)) + let preparedSqlQueries = usePreparedAssetSqlQueries( + () => inferAssetsRes.current?.sql_queries, + () => $workspaceStore + ) + const dispatch = createEventDispatcher() $effect(() => { @@ -156,37 +164,34 @@ dispatch('change', { code, schema }) }) - $effect(() => { - ;[lang, code] - untrack(() => { - inferAssets(lang, code).then((inferAssetsResult) => { - if (inferAssetsResult.status === 'error') return - let newAssets = inferAssetsResult.assets as AssetWithAltAccessType[] - for (const asset of newAssets) { - const old = assets?.find((a) => assetEq(a, asset)) - if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type - } - if (!deepEqual(assets, newAssets)) assets = newAssets - }) + watch( + () => inferAssetsRes.current, + () => { + if (!inferAssetsRes.current || inferAssetsRes.current?.status === 'error') return + let newAssets = inferAssetsRes.current.assets as AssetWithAltAccessType[] + for (const asset of newAssets) { + const old = assets?.find((a) => assetEq(a, asset)) + if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type + } + if (!deepEqual(assets, newAssets)) assets = newAssets + } + ) - if (lang === 'ansible') { - inferAnsibleExecutionMode(code).then((v) => { - if ( - v !== undefined && - (v.delegate_to_git_repo_details === null || - v.delegate_to_git_repo_details.resource !== - ansibleAlternativeExecutionMode?.resource || - v.delegate_to_git_repo_details.playbook !== - ansibleAlternativeExecutionMode?.playbook || - v.delegate_to_git_repo_details.inventories_location !== - ansibleAlternativeExecutionMode?.inventories_location || - v.delegate_to_git_repo_details.commit !== ansibleAlternativeExecutionMode?.commit || - v.git_ssh_identity !== ansibleGitSshIdentity) - ) { - ansibleAlternativeExecutionMode = v.delegate_to_git_repo_details - ansibleGitSshIdentity = v.git_ssh_identity - } - }) + watch([() => code, () => lang], () => { + if (lang !== 'ansible') return + inferAnsibleExecutionMode(code).then((v) => { + if ( + v !== undefined && + (v.delegate_to_git_repo_details === null || + v.delegate_to_git_repo_details.resource !== ansibleAlternativeExecutionMode?.resource || + v.delegate_to_git_repo_details.playbook !== ansibleAlternativeExecutionMode?.playbook || + v.delegate_to_git_repo_details.inventories_location !== + ansibleAlternativeExecutionMode?.inventories_location || + v.delegate_to_git_repo_details.commit !== ansibleAlternativeExecutionMode?.commit || + v.git_ssh_identity !== ansibleGitSshIdentity) + ) { + ansibleAlternativeExecutionMode = v.delegate_to_git_repo_details + ansibleGitSshIdentity = v.git_ssh_identity } }) }) @@ -909,6 +914,7 @@ {fixedOverflowWidgets} {args} {enablePreprocessorSnippet} + preparedAssetsSqlQueries={preparedSqlQueries.current} /> datatype === type) } + +export function sqlDataTypeToJsTypeHeuristic(datatype: string): string { + datatype = datatype.toLowerCase() + if ( + datatype.includes('int') || + datatype === 'decimal' || + datatype === 'numeric' || + datatype === 'float' || + datatype === 'real' || + datatype === 'double' + ) { + return 'number' + } else if ( + datatype === 'varchar' || + datatype === 'char' || + datatype === 'text' || + datatype === 'nvarchar' || + datatype === 'nchar' || + datatype === 'string' + ) { + return 'string' + } else if (datatype === 'boolean' || datatype === 'bool' || datatype === 'bit') { + return 'boolean' + } else if ( + datatype === 'date' || + datatype === 'datetime' || + datatype === 'timestamp' || + datatype === 'timestamptz' + ) { + return 'Date' + } else if (datatype === 'json' || datatype === 'jsonb') { + return 'object' + } else { + return 'any' + } +} diff --git a/frontend/src/lib/components/apps/sharedTypes.ts b/frontend/src/lib/components/apps/sharedTypes.ts index 396558e6ab..369e2246ef 100644 --- a/frontend/src/lib/components/apps/sharedTypes.ts +++ b/frontend/src/lib/components/apps/sharedTypes.ts @@ -1,5 +1,6 @@ import type { Schema } from '$lib/common' import type { Preview } from '$lib/gen' +import type { AssetWithAltAccessType } from '../assets/lib' export type InlineScript = { content: string @@ -11,4 +12,5 @@ export type InlineScript = { refreshOn?: { id: string; key: string }[] suggestedRefreshOn?: { id: string; key: string }[] id?: number + assets?: AssetWithAltAccessType[] } diff --git a/frontend/src/lib/components/common/OnChange.svelte b/frontend/src/lib/components/common/OnChange.svelte index 6460edd1c9..332c05bb4d 100644 --- a/frontend/src/lib/components/common/OnChange.svelte +++ b/frontend/src/lib/components/common/OnChange.svelte @@ -1,6 +1,5 @@ -{#if enableParser} - {#each allModules as mod (mod.id)} - {#if mod.value.type === 'rawscript'} - {@const v = mod.value} - parseAndUpdateRawScriptModule(v)} /> - {/if} - {/each} -{/if} + $effect(() => { + if (!enableParser) return + for (const mod of allModules) { + const modValue = mod.value + if (modValue.type === 'rawscript') { + // Recompute any raw script module assets when its content changes + watch( + [() => modValue.content], + () => { + parseAndUpdateRawScriptModule(modValue, mod.id) + }, + { lazy: true } + ) + + // Also recompute if the module is selected + watch([() => selectedId === mod.id], () => { + if (selectedId === mod.id) parseAndUpdateRawScriptModule(modValue, mod.id) + }) + } + } + }) + {#if flowGraphAssetsCtx} diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index b74f387578..9d7849e40e 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -17,7 +17,7 @@ import { getLatestHashForScript, scriptLangToEditorLang } from '$lib/scripts' import PropPickerWrapper from '../propPicker/PropPickerWrapper.svelte' import { getContext, onDestroy, tick, untrack } from 'svelte' - import type { FlowEditorContext } from '../types' + import type { FlowEditorContext, FlowGraphAssetContext } from '../types' import FlowModuleScript from './FlowModuleScript.svelte' import FlowModuleEarlyStop from './FlowModuleEarlyStop.svelte' import FlowModuleSuspend from './FlowModuleSuspend.svelte' @@ -57,6 +57,7 @@ import { useUiIntent } from '$lib/components/copilot/chat/flow/useUiIntent' import { editor as meditor } from 'monaco-editor' import { DynamicInput } from '$lib/utils' + import { usePreparedAssetSqlQueries } from '$lib/infer.svelte' const { selectionManager, @@ -139,6 +140,7 @@ let scriptProgress = $state(undefined) let assets = $derived((flowModule.value.type === 'rawscript' && flowModule.value.assets) || []) + const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') // UI Intent handling for AI tool control useUiIntent(`flow-${flowModule.id}`, { @@ -345,6 +347,11 @@ function onJobDone() { modulePreviewResultViewer?.getOutputPickerInner()?.setJobPreview() } + + let preparedSqlQueries = usePreparedAssetSqlQueries( + () => flowGraphAssetsCtx?.val.sqlQueries[selectedId], + () => $workspaceStore + ) @@ -502,6 +509,7 @@ )} key={`flow-inline-${$workspaceStore}-${$pathStore}-${flowModule.id}`} moduleId={flowModule.id} + preparedAssetsSqlQueries={preparedSqlQueries.current} /> additionalAssetsMap: Record computeAssetsCount: (asset: Asset) => number + sqlQueries: Record }> export type OutputViewerJob = diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte index 5824150511..23e8ca5aec 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptEditor.svelte @@ -6,7 +6,7 @@ import type { Preview, ScriptLang } from '$lib/gen' import { createEventDispatcher, onMount } from 'svelte' import { Trash2 } from 'lucide-svelte' - import { inferArgs } from '$lib/infer' + import { inferArgs, inferAssets } from '$lib/infer' import type { Schema } from '$lib/common' import Editor from '$lib/components/Editor.svelte' import { emptySchema } from '$lib/utils' @@ -19,6 +19,10 @@ import { computeFields } from '../apps/editor/inlineScriptsPanel/utils' import EditorBar from '../EditorBar.svelte' import { LanguageIcon } from '../common/languageIcons' + import { resource } from 'runed' + import { usePreparedAssetSqlQueries } from '$lib/infer.svelte' + import AssetsDropdownButton from '../assets/AssetsDropdownButton.svelte' + import { workspaceStore } from '$lib/stores' interface Props { inlineScript: (InlineScript & { language: ScriptLang }) | undefined @@ -107,6 +111,18 @@ const dispatch = createEventDispatcher() let width = $state(0) + + let inferAssetsRes = resource( + [() => inlineScript?.language, () => inlineScript?.content], + async () => inlineScript && inferAssets(inlineScript.language, inlineScript.content) + ) + let preparedSqlQueries = usePreparedAssetSqlQueries( + () => inferAssetsRes.current?.sql_queries, + () => $workspaceStore + ) + $effect(() => { + if (inlineScript && inferAssetsRes.current) inlineScript.assets = inferAssetsRes.current?.assets + }) {#if inlineScript} @@ -172,6 +188,11 @@
+
+ {#if inlineScript.assets?.length} + + {/if} +
Promise +} + +/** + * Cached promise for the TypeScript worker client + * We lazily initialize this when first needed + */ +let _workerClient: ((...uris: Uri[]) => Promise) | undefined + +async function getWorkerClient(): Promise<(...uris: Uri[]) => Promise> { + try { + // Get or create the worker client + if (!_workerClient) { + _workerClient = (await languages.typescript.getTypeScriptWorker()) as any + } + return _workerClient! + } catch (error) { + console.error('[SqlTypeService] Failed to get TypeScript worker client:', error) + _workerClient = undefined // Reset on error + throw error + } +} + +export async function waitForWorkerInitialization(fileUri: string): Promise { + const WORKER_INIT_TIMEOUT = 10000 + const MAX_RETRIES = 10 + const RETRY_DELAY = 300 + + if (!fileUri.endsWith('.ts')) fileUri += '.ts' + const uri = Uri.parse(fileUri) + + const startTime = Date.now() + + for (let retries = 0; retries < MAX_RETRIES; retries++) { + try { + let workerClient = await getWorkerClient() + await workerClient(uri) + return true + } catch (error) { + if (retries >= 5) { + console.warn( + `[SqlTypeService] Worker not ready yet for ${uri.toString()}, retrying... (${ + retries + 1 + }/${MAX_RETRIES})` + ) + } + if (Date.now() - startTime > WORKER_INIT_TIMEOUT) { + throw new Error( + `[SqlTypeService] Worker initialization timeout for ${uri.toString()}. Custom method not found after ${WORKER_INIT_TIMEOUT}ms` + ) + } + await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY)) + } + } + + throw new Error( + `[SqlTypeService] Worker initialization failed for ${uri.toString()} after ${MAX_RETRIES} retries.` + ) +} + +/** + * Update SQL query type information in the TypeScript worker + * + * This function sends the parsed SQL query details to the custom TypeScript worker, + * which will then inject type parameters into the code that TypeScript analyzes. + * + * @param fileUri - Monaco URI or string path of the file being edited + * @param queries - Array of SQL query details from the WASM parser + * @returns Promise that resolves when the update is complete + */ +export async function updateSqlQueriesInWorker( + fileUri: string, + queries: InferAssetsSqlQueryDetails[] +): Promise { + try { + if (!fileUri.endsWith('.ts')) fileUri += '.ts' + const uri = Uri.parse(fileUri) + const uriString = uri.toString() + + const workerClient = await getWorkerClient() + const worker = await workerClient(uri) + + if (!worker) { + console.warn(`[SqlTypeService] Couldn't load worker for URI: ${uriString}`) + return + } + + const model = editor.getModel(uri) + if (!model) { + console.warn(`[SqlTypeService] No Monaco model found for URI: ${uriString}`) + return + } + + // Call our custom updateSqlQueries method if it exists + // This method is added by our sqlTypePlugin.worker.js + if (typeof worker.updateSqlQueries === 'function') { + await worker.updateSqlQueries(uriString, queries) + revalidateModel(model) + } else { + console.warn( + '[SqlTypeService] Custom worker method updateSqlQueries not found. Is the custom worker loaded?' + ) + } + } catch (error) { + console.error('[SqlTypeService] Failed to update SQL queries in worker:', error) + // Don't throw - we want to fail gracefully if the worker isn't available + } +} + +// https://stackoverflow.com/questions/56050816/is-there-a-way-to-trigger-validation-manually-in-monaco-editor +// Trick to force re-validation of the model to show updated markers +async function revalidateModel(model: editor.ITextModel) { + if (!model || model.isDisposed()) return + + const getWorker = await languages.typescript.getTypeScriptWorker() + const worker = await getWorker(model.uri) + const diagnostics = ( + await Promise.all([ + worker.getSyntacticDiagnostics(model.uri.toString()), + worker.getSemanticDiagnostics(model.uri.toString()) + ]) + ).reduce((a, it) => a.concat(it)) + + const markers = diagnostics.map((d) => { + const start = model.getPositionAt(d.start ?? 1) + const end = model.getPositionAt((d.start ?? 1) + (d.length ?? 0)) + return { + severity: MarkerSeverity.Error, + startLineNumber: start.lineNumber, + startColumn: start.column, + endLineNumber: end.lineNumber, + endColumn: end.column, + message: flattenDiagnosticMessageText(d.messageText, '\n') + } + }) + const owner = model.getLanguageId() + editor.setModelMarkers(model, owner, markers) +} + +function flattenDiagnosticMessageText( + messageText: string | languages.typescript.DiagnosticMessageChain | undefined, + newLine: string +): string { + if (typeof messageText === 'string') { + return messageText + } else if (messageText === undefined) { + return '' + } else { + let result = '' + let indent = 0 + let stack = [messageText] + while (stack.length > 0) { + let messageText = stack.shift()! + if (indent) { + result += newLine + for (let i = 0; i < indent; i++) { + result += ' ' + } + } + result += messageText.messageText + indent++ + stack.push(...(messageText.next || [])) + } + return result + } +} diff --git a/frontend/src/lib/components/vscode.ts b/frontend/src/lib/components/vscode.ts index 22e955dc5c..91962f0df0 100644 --- a/frontend/src/lib/components/vscode.ts +++ b/frontend/src/lib/components/vscode.ts @@ -58,13 +58,12 @@ export function buildWorkerDefinition() { ) }, typescript: () => { + // Use our custom TypeScript worker with SQL type inference plugin return new Worker( - new URL( - '@codingame/monaco-vscode-standalone-typescript-language-features/worker', - import.meta.url - ), + new URL('../monaco_workers/sqlTypePlugin.worker.js', import.meta.url), { - type: 'module' + type: 'module', + name: 'typescript-sql-aware' } ) }, diff --git a/frontend/src/lib/infer.svelte.ts b/frontend/src/lib/infer.svelte.ts new file mode 100644 index 0000000000..419020631c --- /dev/null +++ b/frontend/src/lib/infer.svelte.ts @@ -0,0 +1,98 @@ +import { JobService } from './gen' +import type { InferAssetsSqlQueryDetails, PreparedAssetsSqlQuery } from './infer' +import { ChangeOnDeepInequality, MapResource } from './svelte5Utils.svelte' +import { sqlDataTypeToJsTypeHeuristic } from './components/apps/components/display/dbtable/utils' +import { chunkBy, clone, getQueryStmtCountHeuristic } from './utils' + +function computeQueryKey(query: InferAssetsSqlQueryDetails, workspace?: string) { + return `${query.source_kind}::${query.source_name}::${query.source_schema}::${workspace}::${query.query_string}` +} + +export function usePreparedAssetSqlQueries( + _getQueries: () => InferAssetsSqlQueryDetails[] | undefined, + getWorkspace: () => string | undefined +): { current: InferAssetsSqlQueryDetails[] | undefined } { + let getQueries = new ChangeOnDeepInequality(_getQueries) + + let map = new MapResource( + () => + Object.fromEntries( + getQueries.value?.map((q) => [computeQueryKey(q, getWorkspace() ?? ''), q]) || [] + ), + async (toFetch) => { + let queries = Object.entries(clone(toFetch)) + // We only support preparing datatable source kinds for now. + queries = queries.filter(([_, q]) => q.source_kind === 'datatable') + // We only support preparing single-statement queries for now. + queries = queries.filter(([_, q]) => getQueryStmtCountHeuristic(q.query_string) === 1) + + if (!queries?.length) return {} + try { + // We chunk by source_name to minimize the number of requests. + // For example if we have 10 queries on the same data table, + // we can prepare them all with a single script. + queries.sort((a, b) => a[1].source_name.localeCompare(b[1].source_name)) + let results = ( + await Promise.all( + chunkBy(queries, ([key, q]) => q.source_name).map(async (chunk) => { + console.log( + 'Preparing chunk of queries:', + chunk.map(([_, q]) => q) + ) + let queryContent = chunk + .flatMap(([key, q]) => [ + q.source_schema ? `SET search_path TO ${q.source_schema};` : 'RESET search_path;', + q.query_string + (q.query_string.trim().endsWith(';') ? '' : ';') + ]) + .join('\n') + queryContent = + '-- prepare\n--result_collection=all_statements_first_row\n' + queryContent + + let res = (await JobService.runScriptPreviewAndWaitResult({ + workspace: getWorkspace()!, + requestBody: { + language: 'postgresql', + content: queryContent, + args: { database: `datatable://${chunk[0][1]?.source_name}` } + } + })) as { error?: string; columns?: { name: string; type: string }[] }[] + + console.log('Prepared query content:', res) + + let res2: [string, PreparedAssetsSqlQuery][] = res.map((r, i) => [ + chunk[i][0], + r.columns + ? { + columns: Object.fromEntries( + r.columns.map(({ name, type }) => [ + name, + sqlDataTypeToJsTypeHeuristic(type) + ]) + ) + } + : { error: r.error ?? "Couldn't prepare query " } + ]) + return res2 + }) + ) + ).flat() + + return Object.fromEntries(results) + } catch (e) { + throw e + } + } + ) + + let extendedQueries = $derived.by(() => + getQueries.value?.map((q) => ({ + ...q, + prepared: map.current?.[computeQueryKey(q, getWorkspace())] + })) + ) + return { + get current() { + return extendedQueries + } + } +} diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 228e68d491..96532916e9 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -90,8 +90,21 @@ async function initWasmRuby() { } type InferAssetsResult = - | { status: 'ok'; assets: AssetWithAccessType[] } - | { status: 'error'; error: string } + | { status: 'ok'; assets: AssetWithAccessType[]; sql_queries?: InferAssetsSqlQueryDetails[] } + | { status: 'error'; error: string; assets?: undefined; sql_queries?: undefined } + +export type InferAssetsSqlQueryDetails = { + query_string: string // SQL query with $1 placeholders for interpolations + span: [number, number] // [start, end] byte positions in source code + source_kind: 'datatable' | 'ducklake' // AssetKind equivalent + source_name: string // e.g., "main", "dt" + source_schema?: string // e.g., "public", optional + prepared?: PreparedAssetsSqlQuery +} + +export type PreparedAssetsSqlQuery = + | { columns: Record } // e.g { id: "number", name: "text" } + | { error: string; columns?: undefined } // error message if preparation failed export async function inferAssets( language: SupportedLanguage | undefined, @@ -101,7 +114,7 @@ export async function inferAssets( if (raw_result.startsWith('err:')) { return { status: 'error', error: raw_result.slice(4).trim() } } - return { status: 'ok', assets: JSON.parse(raw_result) as AssetWithAccessType[] } + return { status: 'ok', ...JSON.parse(raw_result) } } try { diff --git a/frontend/src/lib/monaco_workers/sqlTypePlugin.worker.js b/frontend/src/lib/monaco_workers/sqlTypePlugin.worker.js new file mode 100644 index 0000000000..6cf2333e5a --- /dev/null +++ b/frontend/src/lib/monaco_workers/sqlTypePlugin.worker.js @@ -0,0 +1,789 @@ +/** + * Custom TypeScript Language Service Plugin for SQL Type Inference + * + * This worker extends Monaco's TypeScriptWorker to inject type parameters + * into SQL tagged template literals, enabling proper type checking. + * + * Example transformation: + * User writes: let x = sql`SELECT name FROM user` + * TS sees: let x = sql<{ sql: "SELECT name FROM user" }>`SELECT name FROM user` + */ + +import { + TypeScriptWorker, + ts, + initialize +} from '@codingame/monaco-vscode-standalone-typescript-language-features/worker' + +export function injectSqlTypes(code, queries) { + let transformed = code + let addedOffset = 0 + let offsetMap = {} + queries = queries.filter((query) => query?.prepared?.columns) + for (const query of queries) { + let splitIdx = code?.indexOf('`', query.span[0] - 1) + if (splitIdx === -1 || !splitIdx) continue + let leftPart = transformed?.substring(0, splitIdx + addedOffset) + if (leftPart?.endsWith('>')) continue // User added type params manually + let middlePart = + '<{ ' + + Object.entries(query?.prepared?.columns ?? {}) + .map(([key, type]) => `${JSON.stringify(key)}: ${type}`) + .join('; ') + + ' }>' + let rightPart = transformed?.substring(splitIdx + addedOffset) + + // Store the ORIGINAL position (splitIdx - 1), not the transformed one + offsetMap[splitIdx - 1] = middlePart.length + addedOffset += middlePart.length + transformed = leftPart + middlePart + rightPart + } + return { transformed, offsetMap } +} + +// Extend the TypeScriptWorker class +class SqlAwareTypeScriptWorker extends TypeScriptWorker { + constructor(ctx, createData) { + super(ctx, createData) + + // Map of file URI -> SQL query details + this._sqlQueriesByFile = new Map() + // Map of file URI -> version number (incremented when SQL queries change) + this._fileVersions = new Map() + // Cache of transformed code and offset maps per file version + // Structure: fileUri -> {version, originalText, transformed, offsetMap, offsetMapEntries} + this._transformedCodeCache = new Map() + } + + /** + * Gets or computes the cached transformation result for a file + * This centralizes all transformation logic and caching + * @param {string} fileName - File name + * @returns {Object|null} Cached result with {version, originalText, transformed, offsetMap, offsetMapEntries} + */ + _getTransformResult(fileName) { + const currentVersion = this.getScriptVersion(fileName) + const cached = this._transformedCodeCache.get(fileName) + + // Return cached result if version matches + if (cached && cached.version === currentVersion) { + return cached + } + + // Get original snapshot + const originalSnapshot = super.getScriptSnapshot(fileName) + if (!originalSnapshot) { + return null + } + + const queries = this._sqlQueriesByFile.get(fileName) + if (!queries || queries.length === 0) { + return null + } + + // Compute transformation only once per version + try { + const originalText = originalSnapshot.getText(0, originalSnapshot.getLength()) + const { transformed, offsetMap } = injectSqlTypes(originalText, queries) + + // Pre-compute sorted offset map entries for fast position mapping + const offsetMapEntries = Object.entries(offsetMap).sort((a, b) => Number(a[0]) - Number(b[0])) + + const cacheEntry = { + version: currentVersion, + originalText, + transformed, + offsetMap, + offsetMapEntries + } + + this._transformedCodeCache.set(fileName, cacheEntry) + return cacheEntry + } catch (error) { + console.error('[SqlTypePlugin] Error transforming source:', error) + return null + } + } + + /** + * Override getScriptSnapshot to provide transformed source code with type annotations + * This is called by TypeScript when it needs to read source files + */ + getScriptSnapshot(fileName) { + const cached = this._getTransformResult(fileName) + + if (!cached) { + return super.getScriptSnapshot(fileName) + } + + return ts.typescript.ScriptSnapshot.fromString(cached.transformed) + } + /** + * Maps a position from original code to transformed code + * @param {number} position - Position in original code + * @param {string} fileName - File name + * @returns {number} Position in transformed code + */ + _mapPositionToTransformed(position, fileName) { + const cached = this._getTransformResult(fileName) + if (!cached) { + return position + } + + let cumulativeOffset = 0 + for (const [pos, offset] of cached.offsetMapEntries) { + const originalPos = Number(pos) + // If the position is after this injection point in the original code + if (position > originalPos) { + cumulativeOffset += offset + } else { + break + } + } + + return position + cumulativeOffset + } + + /** + * Maps a position from transformed code back to original code + * @param {number} position - Position in transformed code + * @param {string} fileName - File name + * @returns {number} Position in original code + */ + _mapPositionToOriginal(position, fileName) { + const cached = this._getTransformResult(fileName) + if (!cached) { + return position + } + + let cumulativeOffset = 0 + for (const [pos, offset] of cached.offsetMapEntries) { + const originalPos = Number(pos) + const transformedPos = originalPos + cumulativeOffset + + // If position in transformed code is after this injection point + if (position > transformedPos) { + cumulativeOffset += offset + } else { + break + } + } + + return position - cumulativeOffset + } + + /** + * Override getQuickInfoAtPosition to map hover positions correctly + * This fixes the offset issue when hovering over code + */ + async getQuickInfoAtPosition(fileName, position) { + // Map the position from original code to transformed code + const transformedPosition = this._mapPositionToTransformed(position, fileName) + + // Get quick info from the base class using the transformed position + const quickInfo = await super.getQuickInfoAtPosition(fileName, transformedPosition) + + if (!quickInfo) { + return quickInfo + } + + // Map the text span back to original positions + if (quickInfo.textSpan) { + quickInfo.textSpan.start = this._mapPositionToOriginal(quickInfo.textSpan.start, fileName) + } + + return quickInfo + } + + /** + * Override getCompletionsAtPosition to map autocomplete positions correctly + * This fixes the offset issue when showing autocomplete suggestions + */ + async getCompletionsAtPosition(fileName, position, options) { + // Map the position from original code to transformed code + const transformedPosition = this._mapPositionToTransformed(position, fileName) + + // Get completions from the base class using the transformed position + const completions = await super.getCompletionsAtPosition(fileName, transformedPosition, options) + + if (!completions) { + return completions + } + + // Map all completion entry replacement spans back to original positions + if (completions.entries) { + for (const entry of completions.entries) { + if (entry.replacementSpan) { + entry.replacementSpan.start = this._mapPositionToOriginal( + entry.replacementSpan.start, + fileName + ) + } + } + } + + return completions + } + + /** + * Override getCompletionEntryDetails to map positions in detailed completion info + */ + async getCompletionEntryDetails( + fileName, + position, + entryName, + formatOptions, + source, + preferences, + data + ) { + // Map the position from original code to transformed code + const transformedPosition = this._mapPositionToTransformed(position, fileName) + + // Get details from the base class using the transformed position + const details = await super.getCompletionEntryDetails( + fileName, + transformedPosition, + entryName, + formatOptions, + source, + preferences, + data + ) + + if (!details) { + return details + } + + // Map any code actions back to original positions + if (details.codeActions) { + for (const action of details.codeActions) { + if (action.changes) { + for (const change of action.changes) { + if (change.textChanges) { + for (const textChange of change.textChanges) { + if (textChange.span) { + textChange.span.start = this._mapPositionToOriginal( + textChange.span.start, + fileName + ) + } + } + } + } + } + } + } + + return details + } + + /** + * Maps a DocumentSpan (or derived types) from transformed to original positions + * DocumentSpan is used by DefinitionInfo, ReferenceEntry, ImplementationLocation, etc. + */ + _mapDocumentSpan(span, fileName) { + if (!span) return span + + if (span.textSpan) { + span.textSpan.start = this._mapPositionToOriginal(span.textSpan.start, fileName) + } + if (span.contextSpan) { + span.contextSpan.start = this._mapPositionToOriginal(span.contextSpan.start, fileName) + } + if (span.originalTextSpan) { + span.originalTextSpan.start = this._mapPositionToOriginal( + span.originalTextSpan.start, + fileName + ) + } + if (span.originalContextSpan) { + span.originalContextSpan.start = this._mapPositionToOriginal( + span.originalContextSpan.start, + fileName + ) + } + + return span + } + + /** + * Maps an array of DocumentSpans + */ + _mapDocumentSpans(spans, fileName) { + if (!spans) return spans + return spans.map((span) => this._mapDocumentSpan(span, fileName)) + } + + /** + * Override getDefinitionAtPosition - Go to Definition + */ + async getDefinitionAtPosition(fileName, position) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const definitions = await super.getDefinitionAtPosition(fileName, transformedPosition) + + return this._mapDocumentSpans(definitions, fileName) + } + + /** + * Override getDefinitionAndBoundSpan - Go to Definition with bound span + */ + async getDefinitionAndBoundSpan(fileName, position) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const result = await super.getDefinitionAndBoundSpan(fileName, transformedPosition) + + if (!result) return result + + // Map definitions + if (result.definitions) { + result.definitions = this._mapDocumentSpans(result.definitions, fileName) + } + + // Map text span + if (result.textSpan) { + result.textSpan.start = this._mapPositionToOriginal(result.textSpan.start, fileName) + } + + return result + } + + /** + * Override getTypeDefinitionAtPosition - Go to Type Definition + */ + async getTypeDefinitionAtPosition(fileName, position) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const definitions = await super.getTypeDefinitionAtPosition(fileName, transformedPosition) + + return this._mapDocumentSpans(definitions, fileName) + } + + /** + * Override getImplementationAtPosition - Go to Implementation + */ + async getImplementationAtPosition(fileName, position) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const implementations = await super.getImplementationAtPosition(fileName, transformedPosition) + + return this._mapDocumentSpans(implementations, fileName) + } + + /** + * Override getReferencesAtPosition - Find All References + */ + async getReferencesAtPosition(fileName, position) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const references = await super.getReferencesAtPosition(fileName, transformedPosition) + + return this._mapDocumentSpans(references, fileName) + } + + /** + * Override findReferences - Find References (alternative API) + */ + async findReferences(fileName, position) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const referencedSymbols = await super.findReferences(fileName, transformedPosition) + + if (!referencedSymbols) return referencedSymbols + + // Map each ReferencedSymbol + for (const symbol of referencedSymbols) { + // Map definition + if (symbol.definition) { + this._mapDocumentSpan(symbol.definition, fileName) + } + + // Map references + if (symbol.references) { + symbol.references = this._mapDocumentSpans(symbol.references, fileName) + } + } + + return referencedSymbols + } + + /** + * Override getDocumentHighlights - Highlight occurrences + */ + async getDocumentHighlights(fileName, position, filesToSearch) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const highlights = await super.getDocumentHighlights( + fileName, + transformedPosition, + filesToSearch + ) + + if (!highlights) return highlights + + // Map each DocumentHighlights + for (const highlight of highlights) { + if (highlight.highlightSpans) { + for (const span of highlight.highlightSpans) { + if (span.textSpan) { + span.textSpan.start = this._mapPositionToOriginal(span.textSpan.start, fileName) + } + if (span.contextSpan) { + span.contextSpan.start = this._mapPositionToOriginal(span.contextSpan.start, fileName) + } + } + } + } + + return highlights + } + + /** + * Override getRenameInfo - Check if rename is possible + */ + async getRenameInfo(fileName, position, options) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const renameInfo = await super.getRenameInfo(fileName, transformedPosition, options) + + if (!renameInfo) return renameInfo + + // Map triggerSpan if present + if (renameInfo.triggerSpan) { + renameInfo.triggerSpan.start = this._mapPositionToOriginal( + renameInfo.triggerSpan.start, + fileName + ) + } + + return renameInfo + } + + /** + * Override findRenameLocations - Get all rename locations + */ + async findRenameLocations( + fileName, + position, + findInStrings, + findInComments, + providePrefixAndSuffixTextForRename + ) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const locations = await super.findRenameLocations( + fileName, + transformedPosition, + findInStrings, + findInComments, + providePrefixAndSuffixTextForRename + ) + + return this._mapDocumentSpans(locations, fileName) + } + + /** + * Override getSignatureHelpItems - Parameter hints + */ + async getSignatureHelpItems(fileName, position, options) { + const transformedPosition = this._mapPositionToTransformed(position, fileName) + const signatureHelp = await super.getSignatureHelpItems(fileName, transformedPosition, options) + + if (!signatureHelp) return signatureHelp + + // Map applicableSpan + if (signatureHelp.applicableSpan) { + signatureHelp.applicableSpan.start = this._mapPositionToOriginal( + signatureHelp.applicableSpan.start, + fileName + ) + } + + return signatureHelp + } + + /** + * Maps TextChange objects (used in refactorings and code fixes) + */ + _mapTextChanges(textChanges, fileName) { + if (!textChanges) return textChanges + + for (const change of textChanges) { + if (change.span) { + change.span.start = this._mapPositionToOriginal(change.span.start, fileName) + } + } + + return textChanges + } + + /** + * Maps FileTextChanges (used in refactorings and code fixes) + */ + _mapFileTextChanges(fileTextChanges, fileName) { + if (!fileTextChanges) return fileTextChanges + + for (const fileChange of fileTextChanges) { + if (fileChange.textChanges) { + this._mapTextChanges(fileChange.textChanges, fileName) + } + } + + return fileTextChanges + } + + /** + * Override getApplicableRefactors - Get available refactorings + */ + async getApplicableRefactors( + fileName, + positionOrRange, + preferences, + triggerReason, + kind, + includeInteractiveActions + ) { + // Map position or range to transformed + let transformedPositionOrRange = positionOrRange + if (typeof positionOrRange === 'number') { + transformedPositionOrRange = this._mapPositionToTransformed(positionOrRange, fileName) + } else if (positionOrRange && typeof positionOrRange === 'object') { + // It's a TextRange { pos, end } + transformedPositionOrRange = { + pos: this._mapPositionToTransformed(positionOrRange.pos, fileName), + end: this._mapPositionToTransformed(positionOrRange.end, fileName) + } + } + + const refactors = await super.getApplicableRefactors( + fileName, + transformedPositionOrRange, + preferences, + triggerReason, + kind, + includeInteractiveActions + ) + + // Note: ApplicableRefactorInfo doesn't contain positions, so no mapping needed + return refactors + } + + /** + * Override getEditsForRefactor - Get edits for a specific refactoring + */ + async getEditsForRefactor( + fileName, + formatOptions, + positionOrRange, + refactorName, + actionName, + preferences, + interactiveRefactorArguments + ) { + // Map position or range to transformed + let transformedPositionOrRange = positionOrRange + if (typeof positionOrRange === 'number') { + transformedPositionOrRange = this._mapPositionToTransformed(positionOrRange, fileName) + } else if (positionOrRange && typeof positionOrRange === 'object') { + transformedPositionOrRange = { + pos: this._mapPositionToTransformed(positionOrRange.pos, fileName), + end: this._mapPositionToTransformed(positionOrRange.end, fileName) + } + } + + const refactorEditInfo = await super.getEditsForRefactor( + fileName, + formatOptions, + transformedPositionOrRange, + refactorName, + actionName, + preferences, + interactiveRefactorArguments + ) + + if (!refactorEditInfo) return refactorEditInfo + + // Map edits + if (refactorEditInfo.edits) { + this._mapFileTextChanges(refactorEditInfo.edits, fileName) + } + + // Map renameLocation if present + if (refactorEditInfo.renameLocation) { + refactorEditInfo.renameLocation = this._mapPositionToOriginal( + refactorEditInfo.renameLocation, + fileName + ) + } + + return refactorEditInfo + } + + /** + * Override getCodeFixesAtPosition - Get quick fixes for errors + */ + async getCodeFixesAtPosition(fileName, start, end, errorCodes, formatOptions, preferences) { + const transformedStart = this._mapPositionToTransformed(start, fileName) + const transformedEnd = this._mapPositionToTransformed(end, fileName) + + const codeFixes = await super.getCodeFixesAtPosition( + fileName, + transformedStart, + transformedEnd, + errorCodes, + formatOptions, + preferences + ) + + if (!codeFixes) return codeFixes + + // Map each CodeFixAction + for (const fix of codeFixes) { + if (fix.changes) { + this._mapFileTextChanges(fix.changes, fileName) + } + if (fix.fixAllDescription) { + // fixAllDescription doesn't contain positions + } + } + + return codeFixes + } + + /** + * Maps diagnostics positions from transformed code back to original code + * @param {Array} diagnostics - TypeScript diagnostics + * @param {string} fileName - File name + * @returns {Array} Diagnostics with corrected positions + */ + _mapDiagnostics(diagnostics, fileName) { + try { + return diagnostics.map((diagnostic) => { + if (!diagnostic?.start) return diagnostic + diagnostic.start = this._mapPositionToOriginal(diagnostic.start, fileName) + return diagnostic + }) + } catch (error) { + console.error('[SqlTypePlugin] Error mapping diagnostics:', error) + return diagnostics + } + } + + /** + * Creates diagnostics from SQL query preparation errors + * @param {string} fileName - File name + * @returns {Array} SQL error diagnostics + */ + _createSqlErrorDiagnostics(fileName) { + const queries = this._sqlQueriesByFile.get(fileName) + if (!queries || queries.length === 0) { + return [] + } + + const cached = this._getTransformResult(fileName) + const originalCode = cached?.originalText ?? '' + if (!originalCode) { + // Fallback if no cached result + const originalSnapshot = super.getScriptSnapshot(fileName) + if (!originalSnapshot) { + return [] + } + } + + const sqlDiagnostics = [] + for (const query of queries) { + let messageText = query?.prepared?.error + if (typeof messageText === 'string') { + let queryStartIdx = originalCode.indexOf('`', (query.span?.[0] || 1) - 1) + 1 + // Create a diagnostic error for this query + let prefix = 'Failed to prepare query: db error: ERROR: ' + if (messageText.startsWith(prefix)) messageText = messageText.substring(prefix.length) + const diagnostic = { + code: 'SQL_PREPARATION_ERROR', + category: ts.typescript.DiagnosticCategory.Error, + messageText, + file: fileName, + start: queryStartIdx, + length: query.span?.[1] ? query.span[1] - queryStartIdx - 2 : 0, + source: 'sql' + } + sqlDiagnostics.push(diagnostic) + } + } + + return sqlDiagnostics + } + + async getSyntacticDiagnostics(fileName) { + const diagnostics = await super.getSyntacticDiagnostics(fileName) + return this._mapDiagnostics(diagnostics, fileName) + } + + async getSemanticDiagnostics(fileName) { + const diagnostics = await super.getSemanticDiagnostics(fileName) + const mappedDiagnostics = this._mapDiagnostics(diagnostics, fileName) + const sqlDiagnostics = this._createSqlErrorDiagnostics(fileName) + return [...mappedDiagnostics, ...sqlDiagnostics] + } + + async getSuggestionDiagnostics(fileName) { + const diagnostics = await super.getSuggestionDiagnostics(fileName) + return this._mapDiagnostics(diagnostics, fileName) + } + + async getCompilerOptionsDiagnostics(fileName) { + const diagnostics = await super.getCompilerOptionsDiagnostics(fileName) + return this._mapDiagnostics(diagnostics, fileName) + } + + /** + * Override getScriptVersion to return an incremented version when SQL queries change + * This forces TypeScript to invalidate its cache and re-read the snapshot + */ + getScriptVersion(fileName) { + const baseVersion = super.getScriptVersion(fileName) + const sqlVersion = this._fileVersions.get(fileName) || 0 + // Combine base version with SQL version to create a unique version string + return `${baseVersion}-sql${sqlVersion}` + } + + /** + * Custom method to update SQL query information from the main thread + * This is called via worker messaging from the Editor + * + * @param {string} fileUri - URI of the file being edited + * @param {Array} queries - Array of SQL query details + */ + async updateSqlQueries(fileUri, queries) { + // Invalidate cache when SQL queries change + this._transformedCodeCache.delete(fileUri) + + if (!queries || queries.length === 0) { + this._sqlQueriesByFile.delete(fileUri) + } else { + this._sqlQueriesByFile.set(fileUri, queries) + } + + // Increment the version to force TypeScript to re-read the snapshot + const currentVersion = this._fileVersions.get(fileUri) || 0 + this._fileVersions.set(fileUri, currentVersion + 1) + + return true + } +} + +// Create function that Monaco expects +export function create(ctx, createData) { + return new SqlAwareTypeScriptWorker(ctx, createData) +} + +if (typeof self !== 'undefined' && self) { + // Initialize the worker + self.onmessage = () => { + initialize((ctx, createData) => { + return create(ctx, createData) + }) + } + + // This function is called by Monaco's TypeScript worker if customWorkerPath is used + // It receives the base TypeScriptWorker class, TypeScript API (ts), and libFileMap + // We'll keep this as a fallback + // @ts-ignore + self.customTSWorkerFactory = function (TypeScriptWorkerBase, tsApi, libFileMap) { + return SqlAwareTypeScriptWorker + } + + console.log('[SqlTypePlugin] Worker module loaded') +} diff --git a/frontend/src/lib/monaco_workers/sqlTypePlugin.worker.test.ts b/frontend/src/lib/monaco_workers/sqlTypePlugin.worker.test.ts new file mode 100644 index 0000000000..f0aaf3b0f2 --- /dev/null +++ b/frontend/src/lib/monaco_workers/sqlTypePlugin.worker.test.ts @@ -0,0 +1,105 @@ +import { describe, it, expect } from 'vitest' +import { injectSqlTypes, create } from './sqlTypePlugin.worker.js' + +describe('sqlTypePlugin.worker', () => { + describe('code transformation', () => { + it('inserts type annotations on data table queries', () => { + let { transformed } = injectSqlTypes(testCode, testQueries) + let expected = ` +export async function main() { + let sql = wmill.datatable('main') + error_before_first_transform + let friend = await sql<{ "id": number; "nickname": string }>\`SELECT id, name as nickname FROM friends\`.fetchOne() + error_after_first_transform + let friend2 = await sql<{ "name": string }>\`SELECT name FROM friends\`.fetchOne() + let bait = await not_an_asset\`SELECT 23 FROM friends\`.fetchOne() + let friend3 = await sql<{ "id": number; "nickname": string }>\`SELECT id, name as nickname FROM friends\`.fetchOne() + return friend +} + +error_at_the_end` + expect(transformed).toBe(expected) + }) + }) + + describe('maps from transformed to original (displayed) position', () => { + it("Doesn't change when before first transform", () => { + let tsWorker = createMockTsWorker(testCode, testQueries) + let pos = tsWorker._mapPositionToOriginal(44, 'test.ts') + expect(pos).toEqual(44) + }) + it('Maps correctly after first transform', () => { + let tsWorker = createMockTsWorker(testCode, testQueries) + let pos = tsWorker._mapPositionToOriginal(217, 'test.ts') + expect(pos).toEqual(179) + }) + it('Maps correctly at the end after multiple transforms', () => { + let tsWorker = createMockTsWorker(testCode, testQueries) + let pos = tsWorker._mapPositionToOriginal(533, 'test.ts') + expect(pos).toEqual(437) + }) + }) + + describe('maps from original (displayed) to transformed position', () => { + it("Doesn't change when before first transform", () => { + let tsWorker = createMockTsWorker(testCode, testQueries) + let pos = tsWorker._mapPositionToTransformed(44, 'test.ts') + expect(pos).toEqual(44) + }) + it('Maps correctly after first transform', () => { + let tsWorker = createMockTsWorker(testCode, testQueries) + let pos = tsWorker._mapPositionToTransformed(179, 'test.ts') + expect(pos).toEqual(217) + }) + it('Maps correctly at the end after multiple transforms', () => { + let tsWorker = createMockTsWorker(testCode, testQueries) + let pos = tsWorker._mapPositionToTransformed(437, 'test.ts') + expect(pos).toEqual(533) + }) + }) +}) + +let testCode = ` +export async function main() { + let sql = wmill.datatable('main') + error_before_first_transform + let friend = await sql\`SELECT id, name as nickname FROM friends\`.fetchOne() + error_after_first_transform + let friend2 = await sql\`SELECT name FROM friends\`.fetchOne() + let bait = await not_an_asset\`SELECT 23 FROM friends\`.fetchOne() + let friend3 = await sql\`SELECT id, name as nickname FROM friends\`.fetchOne() + return friend +} + +error_at_the_end` + +let testQueries = [ + { + query_string: 'SELECT id, name as nickname FROM friends', + span: [120, 165], + source_kind: 'datatable', + source_name: 'main', + prepared: { columns: { id: 'number', nickname: 'string' } } + }, + { + query_string: 'SELECT name FROM friends', + span: [229, 258], + source_kind: 'datatable', + source_name: 'main', + prepared: { columns: { name: 'string' } } + }, + { + query_string: 'SELECT id, name as nickname FROM friends', + span: [359, 404], + source_kind: 'datatable', + source_name: 'main', + prepared: { columns: { id: 'number', nickname: 'string' } } + } +] + +function createMockTsWorker(code: string, queries: typeof testQueries) { + let tsWorker = create() + ;(tsWorker as any).setMockCode(code) + tsWorker.updateSqlQueries('test.ts', queries) + return tsWorker +} diff --git a/frontend/src/lib/svelte5Utils.svelte.ts b/frontend/src/lib/svelte5Utils.svelte.ts index 90de70ba6d..1b3300530f 100644 --- a/frontend/src/lib/svelte5Utils.svelte.ts +++ b/frontend/src/lib/svelte5Utils.svelte.ts @@ -3,6 +3,7 @@ import { untrack } from 'svelte' import { deepEqual } from 'fast-equals' import { type StateStore } from './utils' +import { resource, type ResourceReturn } from 'runed' export function withProps(component: Component, props: Props) { const ret = $state({ @@ -109,3 +110,80 @@ export class ChangeTracker { return false } } + +/** + * This allows using async resources that only fetch missing data based on a set of keys. + * It maintains a Record of the fetched data and only calls the fetcher for keys that + * are not already present in the map. + * The fetcher takes a record of keys to allow fetching multiple items in a single call. + */ +export class MapResource { + private _cached: Record = {} + private _fetcherResource: ResourceReturn, unknown, false> + + constructor( + getValues: () => Record, + fetcher: (toFetch: Record) => Promise> + ) { + this._fetcherResource = resource(getValues, async (values) => { + let obj = { ...this._cached } + + // Delete keys that are no longer present. + for (const key of Object.keys(obj)) { + if (!(key in values)) { + delete obj[key] + } + } + + // Determine which keys are missing and need to be fetched + let toFetch: Record = {} + for (const [key, value] of Object.entries(values)) { + if (!obj[key]) { + toFetch[key] = value + } + } + + // Fetch missing data and update the map + if (Object.keys(toFetch).length > 0) { + let fetchedData = await fetcher(toFetch) + for (const key of Object.keys(toFetch)) { + let value = fetchedData[key] + obj[key] = value + } + } + + this._cached = obj + + return { ...obj } + }) + } + + get current(): Record | undefined { + return this._fetcherResource.current + } + + get loading(): boolean { + return this._fetcherResource.loading + } + + get error(): Error | undefined { + return this._fetcherResource.error + } +} + +export class ChangeOnDeepInequality { + private _cached: T | undefined = $state() + + constructor(compute: () => T) { + $effect.pre(() => { + const newVal = compute() + if (!deepEqual(newVal, this._cached)) { + this._cached = newVal + } + }) + } + + get value(): T { + return this._cached! + } +} diff --git a/frontend/src/lib/test-setup.ts b/frontend/src/lib/test-setup.ts index d34a9369af..8966a34f96 100644 --- a/frontend/src/lib/test-setup.ts +++ b/frontend/src/lib/test-setup.ts @@ -2,6 +2,8 @@ * Vitest setup file to mock browser globals for testing */ +import { vi } from 'vitest' + // Mock localStorage const localStorageMock = { store: {} as Record, @@ -57,4 +59,29 @@ Object.defineProperty(globalThis, 'localStorage', { Object.defineProperty(globalThis, 'sessionStorage', { value: sessionStorageMock, writable: true -}) \ No newline at end of file +}) + +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features/worker', () => ({ + TypeScriptWorker: class TypeScriptWorker { + private _mockScriptSnapshot?: { + getText: (start: number, end: number) => string + getLength: () => number + } + + setMockCode(t: string) { + this._mockScriptSnapshot = { + getText: (start: number, end: number) => t.substring(start, end), + getLength: () => t.length + } + } + + getScriptSnapshot(_fileName: string) { + return this._mockScriptSnapshot + } + getScriptVersion(_fileName: string) { + return '1' + } + }, + ts: undefined, + initialize: undefined +})) diff --git a/frontend/src/lib/utils.test.ts b/frontend/src/lib/utils.test.ts new file mode 100644 index 0000000000..d5e1ba0c89 --- /dev/null +++ b/frontend/src/lib/utils.test.ts @@ -0,0 +1,285 @@ +import { describe, it, expect } from 'vitest' +import { getQueryStmtCountHeuristic } from './utils' + +describe('getQueryStmtCountHeuristic', () => { + describe('basic statements', () => { + it('should count a single statement without semicolon', () => { + expect(getQueryStmtCountHeuristic('SELECT * FROM users')).toBe(1) + }) + + it('should count a single statement with semicolon', () => { + expect(getQueryStmtCountHeuristic('SELECT * FROM users;')).toBe(1) + }) + + it('should count multiple statements', () => { + expect(getQueryStmtCountHeuristic('SELECT * FROM users; SELECT * FROM posts;')).toBe(2) + }) + + it('should handle empty string as implicit statement', () => { + expect(getQueryStmtCountHeuristic('')).toBe(0) + }) + + it('should handle whitespace only as implicit statement', () => { + expect(getQueryStmtCountHeuristic(' \n\t ')).toBe(0) + }) + + it('should handle statement with trailing whitespace', () => { + expect(getQueryStmtCountHeuristic('SELECT * FROM users; \n ')).toBe(1) + }) + + it('should count three statements', () => { + expect( + getQueryStmtCountHeuristic('SELECT 1; UPDATE users SET name = "test"; DELETE FROM logs;') + ).toBe(3) + }) + }) + + describe('single-quoted strings', () => { + it('should ignore semicolon in single-quoted string', () => { + expect(getQueryStmtCountHeuristic("SELECT 'hello;world' FROM users")).toBe(1) + }) + + it('should handle escaped single quote', () => { + expect(getQueryStmtCountHeuristic("SELECT 'it''s working' FROM users")).toBe(1) + }) + + it('should handle multiple escaped single quotes', () => { + expect(getQueryStmtCountHeuristic("SELECT 'it''s a ''test''' FROM users")).toBe(1) + }) + + it('should handle semicolon in escaped quote context', () => { + expect(getQueryStmtCountHeuristic("SELECT 'val''s;data' FROM users")).toBe(1) + }) + + it('should handle multiple statements with single-quoted strings', () => { + expect( + getQueryStmtCountHeuristic("SELECT 'hello;' FROM users; SELECT 'world;' FROM posts") + ).toBe(2) + }) + + it('should handle adjacent escaped quotes', () => { + expect(getQueryStmtCountHeuristic("SELECT '''' FROM users")).toBe(1) + }) + }) + + describe('double-quoted strings', () => { + it('should ignore semicolon in double-quoted string', () => { + expect(getQueryStmtCountHeuristic('SELECT "hello;world" FROM users')).toBe(1) + }) + + it('should handle escaped double quote', () => { + expect(getQueryStmtCountHeuristic('SELECT "it""s working" FROM users')).toBe(1) + }) + + it('should handle multiple escaped double quotes', () => { + expect(getQueryStmtCountHeuristic('SELECT "it""s a ""test""" FROM users')).toBe(1) + }) + + it('should handle semicolon in escaped quote context', () => { + expect(getQueryStmtCountHeuristic('SELECT "val""s;data" FROM users')).toBe(1) + }) + + it('should handle adjacent escaped quotes', () => { + expect(getQueryStmtCountHeuristic('SELECT """" FROM users')).toBe(1) + }) + }) + + describe('mixed quotes', () => { + it('should handle both single and double quotes in same query', () => { + expect(getQueryStmtCountHeuristic('SELECT "col;1", \'val;2\' FROM users')).toBe(1) + }) + + it('should handle single quote inside double quote', () => { + expect(getQueryStmtCountHeuristic('SELECT "it\'s;ok" FROM users')).toBe(1) + }) + + it('should handle double quote inside single quote', () => { + expect(getQueryStmtCountHeuristic('SELECT \'"quoted";value\' FROM users')).toBe(1) + }) + + it('should handle complex nested scenarios', () => { + expect( + getQueryStmtCountHeuristic('SELECT "a\'b;c", \'d"e;f\', g FROM users; DELETE FROM logs') + ).toBe(2) + }) + }) + + describe('line comments', () => { + it('should ignore semicolon in line comment', () => { + expect(getQueryStmtCountHeuristic('SELECT 1 -- comment;here\nFROM users')).toBe(1) + }) + + it('should handle multiple line comments', () => { + expect(getQueryStmtCountHeuristic('SELECT 1 -- comment;1\n-- comment;2\nFROM users')).toBe(1) + }) + + it('should handle line comment at end of query', () => { + expect(getQueryStmtCountHeuristic('SELECT * FROM users -- final;comment')).toBe(1) + }) + + it('should handle line comment with semicolon at end', () => { + expect(getQueryStmtCountHeuristic('SELECT * FROM users; -- comment;here')).toBe(1) + }) + + it('should count statements after line comment', () => { + expect(getQueryStmtCountHeuristic('SELECT 1; -- comment\nSELECT 2')).toBe(2) + }) + + it('should handle line comment without newline at end (stays in comment state)', () => { + expect(getQueryStmtCountHeuristic('SELECT 1 FROM users -- comment;here')).toBe(1) + }) + + it('should handle dashes inside string not as comment', () => { + expect(getQueryStmtCountHeuristic("SELECT '--not;comment' FROM users")).toBe(1) + }) + + it('should handle single dash (not a comment)', () => { + expect(getQueryStmtCountHeuristic('SELECT a-b FROM users')).toBe(1) + }) + }) + + describe('block comments', () => { + it('should ignore semicolon in block comment', () => { + expect(getQueryStmtCountHeuristic('SELECT 1 /* comment;here */ FROM users')).toBe(1) + }) + + it('should handle multiline block comment', () => { + expect(getQueryStmtCountHeuristic('SELECT 1 /* comment;\nacross;\nlines */ FROM users')).toBe( + 1 + ) + }) + + it('should handle multiple block comments', () => { + expect( + getQueryStmtCountHeuristic( + 'SELECT /* c1;here */ 1, /* c2;here */ 2 FROM /* c3;here */ users' + ) + ).toBe(1) + }) + + it('should handle nested-looking block comments', () => { + expect( + getQueryStmtCountHeuristic('SELECT 1 /* outer /* inner;here */ still */ FROM users') + ).toBe(1) + }) + + it('should handle block comment at end', () => { + expect(getQueryStmtCountHeuristic('SELECT * FROM users /* final;comment */')).toBe(1) + }) + + it('should count statements after block comment', () => { + expect(getQueryStmtCountHeuristic('SELECT 1; /* comment */ SELECT 2')).toBe(2) + }) + + it('should handle block comment markers in string', () => { + expect(getQueryStmtCountHeuristic("SELECT '/* not;comment */' FROM users")).toBe(1) + }) + + it('should handle unclosed block comment (stays in comment state)', () => { + expect(getQueryStmtCountHeuristic('SELECT 1 /* unclosed;comment')).toBe(1) + }) + + it('should handle slash-star inside string not as comment', () => { + expect(getQueryStmtCountHeuristic("SELECT '/*;not comment' FROM users")).toBe(1) + }) + }) + + describe('mixed comments and strings', () => { + it('should handle comment after string', () => { + expect(getQueryStmtCountHeuristic("SELECT 'value;1' -- comment;2\nFROM users")).toBe(1) + }) + + it('should handle string after comment', () => { + expect(getQueryStmtCountHeuristic("SELECT 1 -- comment\n, 'value;here' FROM users")).toBe(1) + }) + + it('should handle complex mix', () => { + expect( + getQueryStmtCountHeuristic( + 'SELECT "col;1" /* comment;1 */ , \'val;2\' -- comment;2\nFROM users' + ) + ).toBe(1) + }) + + it('should handle quotes in comments', () => { + expect(getQueryStmtCountHeuristic("SELECT 1 /* it's;working */ FROM users")).toBe(1) + }) + it('should handle comment markers in string', () => { + expect(getQueryStmtCountHeuristic("SELECT '--/*;*/' FROM users -- real;comment")).toBe(1) + }) + }) + + describe('edge cases', () => { + it('should handle only semicolons', () => { + expect(getQueryStmtCountHeuristic(';;;')).toBe(3) + }) + + it('should handle semicolons with whitespace', () => { + expect(getQueryStmtCountHeuristic(' ; ; ; ')).toBe(3) + }) + + it('should handle query ending with multiple semicolons', () => { + expect(getQueryStmtCountHeuristic('SELECT 1;;')).toBe(2) + }) + + it('should handle unclosed single quote (stays in quote state)', () => { + expect(getQueryStmtCountHeuristic("SELECT 'unclosed")).toBe(1) + }) + + it('should handle unclosed double quote (stays in quote state)', () => { + expect(getQueryStmtCountHeuristic('SELECT "unclosed')).toBe(1) + }) + + it('should handle unclosed line comment (no newline at end)', () => { + expect(getQueryStmtCountHeuristic('SELECT 1; -- comment;here')).toBe(1) + }) + + it('should handle all types together', () => { + expect( + getQueryStmtCountHeuristic( + `SELECT "col;1", 'val;2' FROM users WHERE x = 1; -- comment;here +/* block;comment */ UPDATE posts SET title = 'new;title'; +DELETE FROM logs -- final;cleanup` + ) + ).toBe(3) + }) + + it('should handle empty statements', () => { + expect(getQueryStmtCountHeuristic(';')).toBe(1) + }) + + it('should handle consecutive escaped quotes at statement boundary', () => { + expect(getQueryStmtCountHeuristic("SELECT '''';")).toBe(1) + }) + + it('should handle alternating quote types', () => { + expect(getQueryStmtCountHeuristic('SELECT "a", \'b\', "c", \'d\' FROM users')).toBe(1) + }) + + it('should handle very long string with semicolons', () => { + const longString = ';'.repeat(1000) + expect(getQueryStmtCountHeuristic(`SELECT '${longString}' FROM users`)).toBe(1) + }) + + it('should handle realistic multiline query', () => { + const query = ` +SELECT + id, + name, + "user;email" -- email column +FROM users +WHERE status = 'active;pending' + AND created_at > '2024-01-01'; + +/* Update user preferences */ +UPDATE user_settings +SET theme = 'dark;mode' +WHERE user_id IN (SELECT id FROM users WHERE name LIKE '%test%'); + +-- Cleanup old data +DELETE FROM logs WHERE timestamp < NOW() - INTERVAL '30 days' + ` + expect(getQueryStmtCountHeuristic(query)).toBe(3) + }) + }) +}) diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index c22b366bf0..b7bbb6218b 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1684,3 +1684,117 @@ export function getJobKindIcon(jobKind: Job['job_kind']) { return Bot } else if (jobKind) return Code } + +export function chunkBy(array: T[], getKey: (key: T) => string): T[][] { + const chunks: T[][] = [] + + for (const item of array) { + const key = getKey(item) + let lastChunk = chunks[chunks.length - 1] + + if (!lastChunk || getKey(lastChunk[0]) !== key) { + lastChunk = [] + chunks.push(lastChunk) + } + + lastChunk.push(item) + } + + return chunks +} + +// AI generated +export function getQueryStmtCountHeuristic(query: string): number { + // Handle empty or whitespace-only strings + if (query.trim() === '') { + return 0 + } + + let count = 0 + let currState: 'normal' | 'single-quote' | 'double-quote' | 'line-comment' | 'block-comment' = + 'normal' + let hasContentAfterLastSemicolon = false + + for (let i = 0; i < query.length; i++) { + const char = query[i] + const nextChar = query[i + 1] + + switch (currState) { + case 'normal': + if (char === "'") { + currState = 'single-quote' + hasContentAfterLastSemicolon = true + } else if (char === '"') { + currState = 'double-quote' + hasContentAfterLastSemicolon = true + } else if (char === '-' && nextChar === '-') { + currState = 'line-comment' + i++ // skip next char + } else if (char === '/' && nextChar === '*') { + currState = 'block-comment' + hasContentAfterLastSemicolon = true + i++ // skip next char + } else if (char === ';') { + count++ + hasContentAfterLastSemicolon = false + } else if (char !== ' ' && char !== '\t' && char !== '\n' && char !== '\r') { + // Non-whitespace character means we have content + hasContentAfterLastSemicolon = true + } + break + + case 'single-quote': + if (char === "'") { + // In SQL, '' is an escaped single quote + if (nextChar === "'") { + i++ // skip the escaped quote + } else { + currState = 'normal' + } + } + break + + case 'double-quote': + if (char === '"') { + // In SQL, "" is an escaped double quote + if (nextChar === '"') { + i++ // skip the escaped quote + } else { + currState = 'normal' + } + } + break + + case 'line-comment': + if (char === '\n') { + currState = 'normal' + } + break + + case 'block-comment': + if (char === '*' && nextChar === '/') { + currState = 'normal' + i++ // skip next char + } + break + } + } + + // Count implicit final statement if: + // 1. We're in normal state and query doesn't end with semicolon, OR + // 2. We're in a quote state (unclosed quote) - there's an implicit statement + // 3. We're in a block comment state - there's an implicit statement before the unclosed comment + // 4. We're in a line comment state and we had content after the last semicolon before entering the comment + const trimmedQuery = query.trimEnd() + if (currState === 'normal' && trimmedQuery !== '' && !trimmedQuery.endsWith(';')) { + count++ + } else if (currState === 'single-quote' || currState === 'double-quote' || currState === 'block-comment') { + // Unclosed quote or unclosed block comment means there's an implicit statement + count++ + } else if (currState === 'line-comment' && hasContentAfterLastSemicolon) { + // Line comment with content before it means there's an implicit statement + count++ + } + + return count +} diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index e5d5b9fe11..09877d8246 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -2279,6 +2279,19 @@ class SqlQuery: """ return self.fetch(result_collection="last_statement_first_row") + def fetch_one_scalar(self): + """Execute query and fetch first row of results. Return result as a scalar value. + + Returns: + First row of query result as a scalar value + """ + return self.fetch(result_collection="last_statement_first_row_scalar") + + def execute(self): + """Execute query and don't return any results. + """ + self.fetch_one() + def infer_sql_type(value) -> str: """ DuckDB executor requires explicit argument types at declaration diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index 986a3b9112..ee833f67a6 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -15,6 +15,6 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 69eff7be0e..637a53bf8f 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -40,4 +40,4 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/client.d.ts b/typescript-client/client.d.ts index 4cdf547ca6..1ab545f3f7 100644 --- a/typescript-client/client.d.ts +++ b/typescript-client/client.d.ts @@ -19,7 +19,12 @@ export { type S3ObjectRecord, type S3ObjectURI, } from "./s3Types"; -export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils"; +export { + datatable, + ducklake, + type SqlTemplateFunction, + type DatatableSqlTemplateFunction, +} from "./sqlUtils"; export type Sql = string; export type Email = string; export type Base64 = string; diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 980aaad9ed..e766f306b2 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -22,7 +22,12 @@ export { type S3ObjectRecord, type S3ObjectURI, } from "./s3Types"; -export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils"; +export { + datatable, + ducklake, + type SqlTemplateFunction, + type DatatableSqlTemplateFunction, +} from "./sqlUtils"; export { AdminService, diff --git a/typescript-client/sqlUtils.d.ts b/typescript-client/sqlUtils.d.ts index 012ebcb38a..fa73465e2b 100644 --- a/typescript-client/sqlUtils.d.ts +++ b/typescript-client/sqlUtils.d.ts @@ -13,41 +13,70 @@ type FetchParams = { resultCollection?: ResultCollectionT; }; -type SqlResult = - ResultCollectionT extends "last_statement_first_row" - ? object - : ResultCollectionT extends "all_statements_first_row" - ? object[] - : ResultCollectionT extends "last_statement_all_rows" - ? object[] - : ResultCollectionT extends "all_statements_all_rows" - ? object[][] - : ResultCollectionT extends "last_statement_all_rows_scalar" - ? any[] - : ResultCollectionT extends "all_statements_all_rows_scalar" - ? any[][] - : ResultCollectionT extends "last_statement_first_row_scalar" - ? any - : ResultCollectionT extends "all_statements_first_row_scalar" - ? any[] - : unknown; -export type SqlStatement = { +type SqlResult< + T, + ResultCollectionT extends ResultCollection +> = ResultCollectionT extends "last_statement_first_row" + ? T | null + : ResultCollectionT extends "all_statements_first_row" + ? T[] + : ResultCollectionT extends "last_statement_all_rows" + ? T[] + : ResultCollectionT extends "all_statements_all_rows" + ? T[][] + : ResultCollectionT extends "last_statement_all_rows_scalar" + ? T[keyof T][] + : ResultCollectionT extends "all_statements_all_rows_scalar" + ? T[keyof T][][] + : ResultCollectionT extends "last_statement_first_row_scalar" + ? T[keyof T] | null + : ResultCollectionT extends "all_statements_first_row_scalar" + ? T[keyof T][] + : unknown; + +export type SqlStatement = { content: string; args: Record; fetch( params?: FetchParams // The union is for auto-completion - ): Promise>; + ): Promise>; fetchOne( params?: Omit, "resultCollection"> - ): Promise>; + ): Promise>; + + /** + * Execute the SQL query and return only the first row as a scalar value + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOneScalar( + params?: Omit< + FetchParams<"last_statement_first_row_scalar">, + "resultCollection" + > + ): Promise>; + + /** + * Execute the SQL query without fetching rows + * @param params - Optional parameters + */ + execute( + params?: Omit< + FetchParams<"last_statement_first_row_scalar">, + "resultCollection" + > + ): Promise; }; export interface SqlTemplateFunction { - (strings: TemplateStringsArray, ...values: any[]): SqlStatement; + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; +} +export interface DatatableSqlTemplateFunction extends SqlTemplateFunction { + query(sql: string, ...params: any[]): SqlStatement; } -export declare function datatable(name: string): SqlTemplateFunction; +export declare function datatable(name: string): DatatableSqlTemplateFunction; export declare function ducklake(name: string): SqlTemplateFunction; diff --git a/typescript-client/sqlUtils.ts b/typescript-client/sqlUtils.ts index 7246e328ab..6bad58a513 100644 --- a/typescript-client/sqlUtils.ts +++ b/typescript-client/sqlUtils.ts @@ -15,28 +15,30 @@ type FetchParams = { resultCollection?: ResultCollectionT; }; -type SqlResult = - ResultCollectionT extends "last_statement_first_row" - ? any - : ResultCollectionT extends "all_statements_first_row" - ? any[] - : ResultCollectionT extends "last_statement_all_rows" - ? any[] - : ResultCollectionT extends "all_statements_all_rows" - ? any[][] - : ResultCollectionT extends "last_statement_all_rows_scalar" - ? any[] - : ResultCollectionT extends "all_statements_all_rows_scalar" - ? any[][] - : ResultCollectionT extends "last_statement_first_row_scalar" - ? any - : ResultCollectionT extends "all_statements_first_row_scalar" - ? any[] - : unknown; +type SqlResult< + T, + ResultCollectionT extends ResultCollection +> = ResultCollectionT extends "last_statement_first_row" + ? T | null + : ResultCollectionT extends "all_statements_first_row" + ? T[] + : ResultCollectionT extends "last_statement_all_rows" + ? T[] + : ResultCollectionT extends "all_statements_all_rows" + ? T[][] + : ResultCollectionT extends "last_statement_all_rows_scalar" + ? T[keyof T][] + : ResultCollectionT extends "all_statements_all_rows_scalar" + ? T[keyof T][][] + : ResultCollectionT extends "last_statement_first_row_scalar" + ? T[keyof T] | null + : ResultCollectionT extends "all_statements_first_row_scalar" + ? T[keyof T][] + : unknown; /** * SQL statement object with query content, arguments, and execution methods */ -export type SqlStatement = { +export type SqlStatement = { /** Raw SQL content with formatted arguments */ content: string; @@ -50,7 +52,7 @@ export type SqlStatement = { */ fetch( params?: FetchParams // The union is for auto-completion - ): Promise>; + ): Promise>; /** * Execute the SQL query and return only the first row @@ -59,18 +61,38 @@ export type SqlStatement = { */ fetchOne( params?: Omit, "resultCollection"> - ): Promise>; + ): Promise>; + + /** + * Execute the SQL query and return only the first row as a scalar value + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOneScalar( + params?: Omit< + FetchParams<"last_statement_first_row_scalar">, + "resultCollection" + > + ): Promise>; + + /** + * Execute the SQL query without fetching rows + * @param params - Optional parameters + */ + execute( + params?: Omit, "resultCollection"> + ): Promise; }; /** * Template tag function for creating SQL statements with parameterized values */ export interface SqlTemplateFunction { - (strings: TemplateStringsArray, ...values: any[]): SqlStatement; + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; } -interface DatatableSqlTemplateFunction extends SqlTemplateFunction { - query(sql: string, ...params: any[]): SqlStatement; +export interface DatatableSqlTemplateFunction extends SqlTemplateFunction { + query(sql: string, ...params: any[]): SqlStatement; } /** @@ -172,7 +194,7 @@ function sqlProviderImpl( workspace: getWorkspace(), requestBody: { args, content, language }, }); - return result as SqlResult; + return result as SqlResult; } catch (e: any) { let err = e; if ( @@ -198,7 +220,13 @@ function sqlProviderImpl( fetch, fetchOne: (params) => fetch({ ...params, resultCollection: "last_statement_first_row" }), - } satisfies SqlStatement; + fetchOneScalar: (params) => + fetch({ + ...params, + resultCollection: "last_statement_first_row_scalar", + }), + execute: (params) => fetch(params), + } satisfies SqlStatement; }; if (provider === "datatable") { (sqlFn as DatatableSqlTemplateFunction).query = (