mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-21 16:02:28 +00:00
feat: unsafe parameters for sql queries (table names, column names) (#5488)
* Make schema validation struct Schema Validation rules that are constructed from the schema or from the MainArgSig(TODO). * Make other validator builder * Fail dependency job like with lockfile failing for schema validator * Add last types + tests * Remove unused dependency * fix typos * Migration ID was colliding with another, changed it manually * Add Oneof + other fixes * fix: cache for querying scripts correclty handles ScriptMetadata * Add cache for schema validation from main arg sig * Prepare sqlx * Remove default features * Feature flags * WIP: unsafe sql params for sql langauges * Fix down migration table name * cleanup: put validation logic inside a function * Refactor to cache the should_validate boolean Changed the schemavalidators cache to take in an Option<SchemaValidator>, effectively storing the `should_validate_schema` information. Also pass the schema when avaialble to construct the schema validator * Add other job kinds to u8 cache key just in case * Change sql languages to all get arguments as Values instead of RawValue * Only cache if not preview * Add last sql languages and some CI fixes * Rename after typo on `sanitized` * Finish rename * Remove unused import * Fix wrong test * Add newly published regex parser version * Remove default features from cargo.toml * Change to a cleaner syntax for the interpolated args * Update republished parser
This commit is contained in:
@@ -16,6 +16,9 @@ use std::{
|
||||
};
|
||||
pub use windmill_parser::{Arg, MainArgSignature, Typ};
|
||||
|
||||
pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__";
|
||||
pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__";
|
||||
|
||||
pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
|
||||
let parsed = parse_mysql_file(&code)?;
|
||||
if let Some(x) = parsed {
|
||||
@@ -147,7 +150,7 @@ lazy_static::lazy_static! {
|
||||
|
||||
// -- $1 name (type) = default
|
||||
static ref RE_ARG_MYSQL: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
pub static ref RE_ARG_MYSQL_NAMED: Regex = Regex::new(r#"(?m)^-- :([a-z_][a-z0-9_]*) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
pub static ref RE_ARG_MYSQL_NAMED: Regex = Regex::new(r#"(?m)^-- :([a-z_][a-z0-9_]*) \((\w+(?:\([\w, ]+\))?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
|
||||
static ref RE_ARG_PGSQL: Regex = Regex::new(r#"(?m)^-- \$(\d+) (\w+)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
|
||||
@@ -159,6 +162,9 @@ lazy_static::lazy_static! {
|
||||
|
||||
static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
|
||||
// used for `unsafe` sql interpolation
|
||||
// -- %%name%% (type) = default
|
||||
static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%\s*([\s\w\/]+)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
|
||||
}
|
||||
|
||||
fn parsed_default(parsed_typ: &Typ, default: String) -> Option<serde_json::Value> {
|
||||
@@ -225,9 +231,34 @@ fn parse_oracledb_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
}
|
||||
}
|
||||
|
||||
args.append(&mut parse_sql_sanitized_interpolation(code));
|
||||
Ok(Some(args))
|
||||
}
|
||||
|
||||
fn parse_sql_sanitized_interpolation(code: &str) -> Vec<Arg> {
|
||||
let mut args: Vec<Arg> = vec![];
|
||||
|
||||
for cap in RE_ARG_SQL_INTERPOLATION.captures_iter(code) {
|
||||
let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap();
|
||||
let typ = cap.get(2).map(|x| x.as_str());
|
||||
let default = cap.get(3).map(|x| x.as_str().to_string());
|
||||
let has_default = default.is_some();
|
||||
let (parsed_typ, otyp) = parse_unsafe_typ(typ);
|
||||
|
||||
let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
|
||||
args.push(Arg {
|
||||
name,
|
||||
typ: parsed_typ,
|
||||
default: parsed_default,
|
||||
otyp: Some(otyp.to_string()),
|
||||
has_default,
|
||||
oidx: None,
|
||||
});
|
||||
}
|
||||
|
||||
args
|
||||
}
|
||||
|
||||
fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
let mut args: Vec<Arg> = vec![];
|
||||
|
||||
@@ -279,6 +310,7 @@ fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
}
|
||||
}
|
||||
|
||||
args.append(&mut parse_sql_sanitized_interpolation(code));
|
||||
Ok(Some(args))
|
||||
}
|
||||
|
||||
@@ -431,6 +463,7 @@ fn parse_pg_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
}
|
||||
}
|
||||
|
||||
args.append(&mut parse_sql_sanitized_interpolation(code));
|
||||
Ok(Some(args))
|
||||
}
|
||||
|
||||
@@ -485,6 +518,7 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
});
|
||||
}
|
||||
|
||||
args.append(&mut parse_sql_sanitized_interpolation(code));
|
||||
Ok(Some(args))
|
||||
}
|
||||
|
||||
@@ -513,6 +547,7 @@ fn parse_snowflake_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
});
|
||||
}
|
||||
|
||||
args.append(&mut parse_sql_sanitized_interpolation(code));
|
||||
Ok(Some(args))
|
||||
}
|
||||
|
||||
@@ -541,9 +576,25 @@ fn parse_mssql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
});
|
||||
}
|
||||
|
||||
args.append(&mut parse_sql_sanitized_interpolation(code));
|
||||
Ok(Some(args))
|
||||
}
|
||||
|
||||
fn parse_unsafe_typ(typ: Option<&str>) -> (Typ, &'static str) {
|
||||
match typ {
|
||||
Some(s) => {
|
||||
let variants = s
|
||||
.split("/")
|
||||
.map(|x| x.trim().to_string())
|
||||
.filter(|x| !x.is_empty())
|
||||
.collect();
|
||||
|
||||
(Typ::Str(Some(variants)), SANITIZED_ENUM_STR)
|
||||
}
|
||||
None => (Typ::Str(None), SANITIZED_RAW_STRING_STR),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_mysql_typ(typ: &str) -> Typ {
|
||||
match typ {
|
||||
"varchar" | "char" | "binary" | "varbinary" | "blob" | "text" | "enum" | "set" => {
|
||||
@@ -1048,6 +1099,55 @@ SELECT @P2;
|
||||
}
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
#[test]
|
||||
fn test_parse_oracledb_sig() -> anyhow::Result<()> {
|
||||
let code = r#"
|
||||
-- :name1 (int) = 3
|
||||
-- :name2 (text)
|
||||
-- :name4 (text)
|
||||
SELECT :name, :name2;
|
||||
SELECT * FROM table_name WHERE thing = :name4;
|
||||
"#;
|
||||
|
||||
println!("{:#?}", parse_oracledb_sig(code)?);
|
||||
assert_eq!(
|
||||
parse_oracledb_sig(code)?,
|
||||
MainArgSignature {
|
||||
star_args: false,
|
||||
star_kwargs: false,
|
||||
args: vec![
|
||||
Arg {
|
||||
otyp: Some("int".to_string()),
|
||||
name: "name1".to_string(),
|
||||
typ: Typ::Int,
|
||||
default: Some(json!(3)),
|
||||
has_default: true,
|
||||
oidx: None,
|
||||
},
|
||||
Arg {
|
||||
otyp: Some("text".to_string()),
|
||||
name: "name2".to_string(),
|
||||
typ: Typ::Str(None),
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None,
|
||||
},
|
||||
Arg {
|
||||
otyp: Some("text".to_string()),
|
||||
name: "name4".to_string(),
|
||||
typ: Typ::Str(None),
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None,
|
||||
},
|
||||
],
|
||||
no_main_func: None,
|
||||
has_preprocessor: None
|
||||
}
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ use serde::Deserialize;
|
||||
|
||||
use crate::common::{build_http_client, OccupancyMetrics};
|
||||
use crate::handle_child::run_future_with_polling_update_job_poller;
|
||||
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
|
||||
use crate::{
|
||||
common::{build_args_values, resolve_job_timeout},
|
||||
AuthedClientBackgroundTask,
|
||||
@@ -262,15 +263,21 @@ pub async fn do_bigquery(
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
|
||||
|
||||
let queries = parse_sql_blocks(query);
|
||||
|
||||
let mut statement_values: HashMap<String, Value> = HashMap::new();
|
||||
|
||||
let sig = parse_bigquery_sig(&query)
|
||||
.map_err(|x| Error::ExecutionErr(x.to_string()))?
|
||||
.args;
|
||||
|
||||
let (query, args_to_skip) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &bigquery_args)?;
|
||||
|
||||
let queries = parse_sql_blocks(query);
|
||||
|
||||
let mut statement_values: HashMap<String, Value> = HashMap::new();
|
||||
|
||||
|
||||
for arg in &sig {
|
||||
if args_to_skip.contains(&arg.name) {
|
||||
continue;
|
||||
}
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "string".to_string());
|
||||
let arg_n = arg.clone().name;
|
||||
let arg_v = bigquery_args.get(&arg.name).cloned().unwrap_or(json!(""));
|
||||
|
||||
@@ -43,6 +43,7 @@ mod rust_executor;
|
||||
mod worker;
|
||||
mod worker_flow;
|
||||
mod worker_lockfiles;
|
||||
mod sanitized_sql_params;
|
||||
mod schema;
|
||||
|
||||
pub use worker::*;
|
||||
|
||||
@@ -17,6 +17,7 @@ use windmill_queue::{append_logs, CanceledBy};
|
||||
|
||||
use crate::common::{build_args_values, OccupancyMetrics};
|
||||
use crate::handle_child::run_future_with_polling_update_job_poller;
|
||||
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
|
||||
use crate::AuthedClientBackgroundTask;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -132,8 +133,13 @@ pub async fn do_mssql(
|
||||
.map_err(|x| Error::ExecutionErr(x.to_string()))?
|
||||
.args;
|
||||
|
||||
let (query, args_to_skip) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &mssql_args)?;
|
||||
|
||||
let mut prepared_query = Query::new(query.to_owned());
|
||||
for arg in &sig {
|
||||
if args_to_skip.contains(&arg.name) {
|
||||
continue;
|
||||
}
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "string".to_string());
|
||||
let arg_v = mssql_args
|
||||
.get(&arg.name)
|
||||
|
||||
@@ -8,7 +8,6 @@ use mysql_async::{
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, value::RawValue, Value};
|
||||
use sqlx::types::Json;
|
||||
use tokio::sync::Mutex;
|
||||
use windmill_common::{
|
||||
error::{to_anyhow, Error},
|
||||
@@ -22,9 +21,7 @@ use windmill_queue::CanceledBy;
|
||||
use windmill_queue::MiniPulledJob;
|
||||
|
||||
use crate::{
|
||||
common::{build_args_map, OccupancyMetrics},
|
||||
handle_child::run_future_with_polling_update_job_poller,
|
||||
AuthedClientBackgroundTask,
|
||||
common::{build_args_values, OccupancyMetrics}, handle_child::run_future_with_polling_update_job_poller, sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args, AuthedClientBackgroundTask
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -113,36 +110,25 @@ pub async fn do_mysql(
|
||||
column_order: &mut Option<Vec<String>>,
|
||||
occupancy_metrics: &mut OccupancyMetrics,
|
||||
) -> windmill_common::error::Result<Box<RawValue>> {
|
||||
let args = build_args_map(job, client, db).await?.map(Json);
|
||||
let job_args = if args.is_some() {
|
||||
args.as_ref()
|
||||
} else {
|
||||
job.args.as_ref()
|
||||
};
|
||||
let job_args = build_args_values(job, client, db).await?;
|
||||
|
||||
let inline_db_res_path = parse_db_resource(&query);
|
||||
|
||||
let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
|
||||
let val = client
|
||||
Some(client
|
||||
.get_authed()
|
||||
.await
|
||||
.get_resource_value_interpolated::<serde_json::Value>(
|
||||
&inline_db_res_path,
|
||||
Some(job.id.to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let as_raw = serde_json::from_value(val).map_err(|e| {
|
||||
Error::internal_err(format!("Error while parsing inline resource: {e:#}"))
|
||||
})?;
|
||||
|
||||
Some(as_raw)
|
||||
.await?)
|
||||
} else {
|
||||
job_args.and_then(|x| x.get("database").cloned())
|
||||
job_args.get("database").cloned()
|
||||
};
|
||||
|
||||
let database = if let Some(db) = db_arg {
|
||||
serde_json::from_str::<MysqlDatabase>(db.get())
|
||||
serde_json::from_value::<MysqlDatabase>(db)
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?
|
||||
} else {
|
||||
return Err(Error::BadRequest("Missing database argument".to_string()));
|
||||
@@ -171,6 +157,8 @@ pub async fn do_mysql(
|
||||
.map_err(|x| Error::ExecutionErr(x.to_string()))?
|
||||
.args;
|
||||
|
||||
let (query, args_to_skip) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?;
|
||||
|
||||
let using_named_params = RE_ARG_MYSQL_NAMED.captures_iter(query).count() > 0;
|
||||
|
||||
let mut statement_values: Params = match using_named_params {
|
||||
@@ -178,18 +166,16 @@ pub async fn do_mysql(
|
||||
false => Params::Positional(vec![]),
|
||||
};
|
||||
for arg in &sig {
|
||||
if args_to_skip.contains(&arg.name) {
|
||||
continue;
|
||||
}
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "text".to_string());
|
||||
let arg_n = arg.name.clone();
|
||||
let mysql_v = match job_args
|
||||
.and_then(|x| {
|
||||
x.get(arg.name.as_str())
|
||||
.map(|x| serde_json::from_str::<serde_json::Value>(x.get()).ok())
|
||||
})
|
||||
.flatten()
|
||||
.unwrap_or_else(|| json!(null))
|
||||
let mysql_v = match job_args.get(arg.name.as_str())
|
||||
.unwrap_or_else(|| &json!(null))
|
||||
{
|
||||
Value::Null => mysql_async::Value::NULL,
|
||||
Value::Bool(b) => mysql_async::Value::Int(if b { 1 } else { 0 }),
|
||||
Value::Bool(b) => mysql_async::Value::Int(if *b { 1 } else { 0 }),
|
||||
Value::String(s)
|
||||
if arg_t == "timestamp"
|
||||
|| arg_t == "datetime"
|
||||
|
||||
@@ -8,7 +8,6 @@ use itertools::Itertools;
|
||||
use oracle::sql_type::{InnerValue, OracleType, ToSql};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, value::RawValue, Value};
|
||||
use sqlx::types::Json;
|
||||
use windmill_common::{
|
||||
error::{to_anyhow, Error},
|
||||
worker::to_raw_value,
|
||||
@@ -21,8 +20,9 @@ use windmill_parser_sql::{
|
||||
use windmill_queue::CanceledBy;
|
||||
|
||||
use crate::{
|
||||
common::{build_args_map, check_executor_binary_exists, OccupancyMetrics},
|
||||
common::{build_args_values, check_executor_binary_exists, OccupancyMetrics},
|
||||
handle_child::run_future_with_polling_update_job_poller,
|
||||
sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args,
|
||||
AuthedClientBackgroundTask,
|
||||
};
|
||||
|
||||
@@ -221,24 +221,23 @@ fn convert_oracledb_value_to_json(v: &oracle::SqlValue, c: &OracleType) -> serde
|
||||
|
||||
fn get_statement_values(
|
||||
sig: Vec<Arg>,
|
||||
job_args: Option<&Json<HashMap<String, Box<RawValue>>>>,
|
||||
job_args: &HashMap<String, Value>,
|
||||
args_to_skip: &Vec<String>,
|
||||
) -> (Vec<(String, Box<dyn ToSql + Send + Sync>)>, Vec<String>) {
|
||||
let mut statement_values = vec![];
|
||||
let mut errors = vec![];
|
||||
|
||||
for arg in &sig {
|
||||
if args_to_skip.contains(&arg.name) {
|
||||
continue;
|
||||
}
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "text".to_string());
|
||||
let arg_n = arg.name.clone();
|
||||
let oracle_v: Box<dyn ToSql + Send + Sync> = match job_args
|
||||
.and_then(|x| {
|
||||
x.get(arg.name.as_str())
|
||||
.map(|x| serde_json::from_str::<serde_json::Value>(x.get()).ok())
|
||||
})
|
||||
.flatten()
|
||||
.unwrap_or_else(|| json!(null))
|
||||
let oracle_v: Box<dyn ToSql + Send + Sync> = match job_args.get(arg.name.as_str())
|
||||
.unwrap_or_else(|| &json!(null))
|
||||
{
|
||||
// Value::Null => todo!(),
|
||||
Value::Bool(b) => Box::new(b),
|
||||
Value::Bool(b) => Box::new(*b),
|
||||
Value::String(s)
|
||||
if arg_t == "timestamp"
|
||||
|| arg_t == "datetime"
|
||||
@@ -248,10 +247,10 @@ fn get_statement_values(
|
||||
if let Ok(d) = chrono::DateTime::<Utc>::from_str(s.as_str()) {
|
||||
Box::new(d)
|
||||
} else {
|
||||
Box::new(s)
|
||||
Box::new(s.clone())
|
||||
}
|
||||
}
|
||||
Value::String(s) => Box::new(s),
|
||||
Value::String(s) => Box::new(s.clone()),
|
||||
Value::Number(n)
|
||||
if n.is_i64()
|
||||
&& (arg_t == "int"
|
||||
@@ -309,36 +308,27 @@ pub async fn do_oracledb(
|
||||
"Oracle Database",
|
||||
)?;
|
||||
|
||||
let args = build_args_map(job, client, db).await?.map(Json);
|
||||
let job_args = if args.is_some() {
|
||||
args.as_ref()
|
||||
} else {
|
||||
job.args.as_ref()
|
||||
};
|
||||
let job_args = build_args_values(job, client, db).await?;
|
||||
|
||||
let inline_db_res_path = parse_db_resource(&query);
|
||||
|
||||
let db_arg = if let Some(inline_db_res_path) = inline_db_res_path {
|
||||
let val = client
|
||||
.get_authed()
|
||||
.await
|
||||
.get_resource_value_interpolated::<serde_json::Value>(
|
||||
&inline_db_res_path,
|
||||
Some(job.id.to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let as_raw = serde_json::from_value(val).map_err(|e| {
|
||||
Error::internal_err(format!("Error while parsing inline resource: {e:#}"))
|
||||
})?;
|
||||
|
||||
Some(as_raw)
|
||||
Some(
|
||||
client
|
||||
.get_authed()
|
||||
.await
|
||||
.get_resource_value_interpolated::<serde_json::Value>(
|
||||
&inline_db_res_path,
|
||||
Some(job.id.to_string()),
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
job_args.and_then(|x| x.get("database").cloned())
|
||||
job_args.get("database").cloned()
|
||||
};
|
||||
|
||||
let database = if let Some(db) = db_arg {
|
||||
serde_json::from_str::<OracleDatabase>(db.get())
|
||||
serde_json::from_value::<OracleDatabase>(db)
|
||||
.map_err(|e| Error::ExecutionErr(e.to_string()))?
|
||||
} else {
|
||||
return Err(Error::BadRequest("Missing database argument".to_string()));
|
||||
@@ -350,7 +340,9 @@ pub async fn do_oracledb(
|
||||
.map_err(|x| Error::ExecutionErr(x.to_string()))?
|
||||
.args;
|
||||
|
||||
let (statement_values, errors) = get_statement_values(sig.clone(), job_args);
|
||||
let (query, args_to_skip) = sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?;
|
||||
|
||||
let (statement_values, errors) = get_statement_values(sig.clone(), &job_args, &args_to_skip);
|
||||
|
||||
if !errors.is_empty() {
|
||||
return Err(Error::ExecutionErr(errors.join("\n")));
|
||||
@@ -372,13 +364,13 @@ pub async fn do_oracledb(
|
||||
|
||||
let conn_a = Arc::new(std::sync::Mutex::new(conn));
|
||||
|
||||
let queries = parse_sql_blocks(query);
|
||||
let queries = parse_sql_blocks(&query);
|
||||
|
||||
let result_f = if queries.len() > 1 {
|
||||
let f = async {
|
||||
let mut res: Vec<Box<RawValue>> = vec![];
|
||||
for (i, q) in queries.iter().enumerate() {
|
||||
let (vals, _) = get_statement_values(sig.clone(), job_args);
|
||||
let (vals, _) = get_statement_values(sig.clone(), &job_args, &args_to_skip);
|
||||
let r = do_oracledb_inner(
|
||||
q,
|
||||
vals,
|
||||
@@ -399,7 +391,7 @@ pub async fn do_oracledb(
|
||||
|
||||
f.boxed()
|
||||
} else {
|
||||
do_oracledb_inner(query, statement_values, conn_a, Some(column_order), false)?
|
||||
do_oracledb_inner(&query, statement_values, conn_a, Some(column_order), false)?
|
||||
};
|
||||
|
||||
let result = run_future_with_polling_update_job_poller(
|
||||
|
||||
@@ -36,6 +36,7 @@ use windmill_queue::{CanceledBy, MiniPulledJob};
|
||||
|
||||
use crate::common::{build_args_values, sizeof_val, OccupancyMetrics};
|
||||
use crate::handle_child::run_future_with_polling_update_job_poller;
|
||||
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
|
||||
use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE};
|
||||
use bytes::Buf;
|
||||
use lazy_static::lazy_static;
|
||||
@@ -290,6 +291,11 @@ pub async fn do_postgresql(
|
||||
(Some((client, handle)), None)
|
||||
};
|
||||
|
||||
let sig = parse_pgsql_sig(&query).map_err(|x| Error::ExecutionErr(x.to_string()))?;
|
||||
|
||||
let (query, _) =
|
||||
&sanitize_and_interpolate_unsafe_sql_args(query, &sig.args, &pg_args)?;
|
||||
|
||||
let queries = parse_sql_blocks(query);
|
||||
|
||||
let (client, handle) = if let Some((client, handle)) = new_client.as_ref() {
|
||||
@@ -299,7 +305,6 @@ pub async fn do_postgresql(
|
||||
(client, None)
|
||||
};
|
||||
|
||||
let sig = parse_pgsql_sig(&query).map_err(|x| Error::ExecutionErr(x.to_string()))?;
|
||||
let param_idx_to_arg_and_value = sig
|
||||
.args
|
||||
.iter()
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
use anyhow::anyhow;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::Value;
|
||||
use windmill_common::error;
|
||||
use windmill_parser::Arg;
|
||||
use windmill_parser_sql::{SANITIZED_ENUM_STR, SANITIZED_RAW_STRING_STR};
|
||||
|
||||
/// Identifier must be a continuous ASCII alphanumeric word, not starting with
|
||||
/// a number, that can contain underscores
|
||||
fn sanitize_identifier(arg: &Arg, input: &str) -> Result<(), error::Error> {
|
||||
if input.is_empty() {
|
||||
return Err(error::Error::BadRequest(format!("Interpolated argument `{}` cannot be empty", arg.name)));
|
||||
}
|
||||
if input
|
||||
.chars()
|
||||
.next()
|
||||
.map(|c| c.is_ascii_alphabetic())
|
||||
.unwrap_or(false)
|
||||
&& input.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err(error::Error::BadRequest(format!("Interpolated argument `{}` contained forbidden characters. Received `{}` but should only contain alphanumerical characters and `_`.", arg.name, input)))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sanitize_and_interpolate_unsafe_sql_args(
|
||||
code: &str,
|
||||
args: &Vec<Arg>,
|
||||
args_map: &HashMap<String, Value>,
|
||||
) -> Result<(String, Vec<String>), error::Error> {
|
||||
let mut ret = code.to_string();
|
||||
let mut args_to_skip = vec![];
|
||||
|
||||
for arg in args {
|
||||
if let Some(typ) = &arg.otyp {
|
||||
let pattern = format!("%%{}%%", arg.name);
|
||||
match typ.as_str() {
|
||||
SANITIZED_ENUM_STR => {
|
||||
let replace =
|
||||
args_map
|
||||
.get(&arg.name)
|
||||
.and_then(|rv| rv.as_str())
|
||||
.ok_or(anyhow!(
|
||||
"Sanitized enum `{}` needs to receive a string",
|
||||
arg.name
|
||||
))?;
|
||||
let windmill_parser::Typ::Str(Some(variants)) = &arg.typ else {
|
||||
return Err(error::Error::ArgumentErr(format!(
|
||||
"Wrong type of argument for sanitized enum `{}`",
|
||||
arg.name
|
||||
)));
|
||||
};
|
||||
if variants.iter().all(|v| v != replace) {
|
||||
return Err(error::Error::ArgumentErr(format!(
|
||||
"Sanitized enum argument `{}` expected one of `[{}]` but received `{}`",
|
||||
arg.name,
|
||||
variants
|
||||
.iter()
|
||||
.map(|s| format!("{s}"))
|
||||
.collect::<Vec<String>>()
|
||||
.join(","),
|
||||
replace,
|
||||
)));
|
||||
}
|
||||
|
||||
sanitize_identifier(&arg, replace)?;
|
||||
ret = ret.replace(&pattern, replace);
|
||||
args_to_skip.push(arg.name.to_string());
|
||||
}
|
||||
SANITIZED_RAW_STRING_STR => {
|
||||
let replace =
|
||||
args_map
|
||||
.get(&arg.name)
|
||||
.and_then(|rv| rv.as_str())
|
||||
.ok_or(anyhow!(
|
||||
"Sanitized raw string `{}` needs to receive a string",
|
||||
arg.name
|
||||
))?;
|
||||
let windmill_parser::Typ::Str(_) = &arg.typ else {
|
||||
return Err(error::Error::ArgumentErr(format!(
|
||||
"Wrong type of argument for sanitized raw string `{}`",
|
||||
arg.name
|
||||
)));
|
||||
};
|
||||
sanitize_identifier(&arg, replace)?;
|
||||
ret = ret.replace(&pattern, &replace);
|
||||
args_to_skip.push(arg.name.to_string());
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((ret, args_to_skip))
|
||||
}
|
||||
@@ -18,6 +18,7 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::common::{build_http_client, resolve_job_timeout, OccupancyMetrics};
|
||||
use crate::handle_child::run_future_with_polling_update_job_poller;
|
||||
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
|
||||
use crate::{common::build_args_values, AuthedClientBackgroundTask};
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -123,15 +124,21 @@ fn do_snowflake_inner<'a>(
|
||||
skip_collect: bool,
|
||||
http_client: &'a Client,
|
||||
) -> windmill_common::error::Result<BoxFuture<'a, windmill_common::error::Result<Box<RawValue>>>> {
|
||||
body.insert("statement".to_string(), json!(query));
|
||||
|
||||
let mut bindings = serde_json::Map::new();
|
||||
let sig = parse_snowflake_sig(&query)
|
||||
.map_err(|x| Error::ExecutionErr(x.to_string()))?
|
||||
.args;
|
||||
|
||||
let (query, args_to_skip) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?;
|
||||
|
||||
body.insert("statement".to_string(), json!(query));
|
||||
|
||||
let mut bindings = serde_json::Map::new();
|
||||
|
||||
let mut i = 1;
|
||||
for arg in &sig {
|
||||
if args_to_skip.contains(&arg.name) {
|
||||
continue;
|
||||
}
|
||||
let arg_t = arg.otyp.clone().unwrap_or_else(|| "string".to_string());
|
||||
let arg_v = job_args.get(&arg.name).cloned().unwrap_or(json!(""));
|
||||
let snowflake_v = convert_typ_val(arg_t, arg_v);
|
||||
|
||||
Generated
+4
-4
@@ -72,7 +72,7 @@
|
||||
"windmill-parser-wasm-nu": "^1.474.1",
|
||||
"windmill-parser-wasm-php": "^1.429.0",
|
||||
"windmill-parser-wasm-py": "^1.477.1",
|
||||
"windmill-parser-wasm-regex": "^1.439.0",
|
||||
"windmill-parser-wasm-regex": "^1.478.2",
|
||||
"windmill-parser-wasm-rust": "^1.429.0",
|
||||
"windmill-parser-wasm-ts": "^1.429.0",
|
||||
"windmill-parser-wasm-yaml": "^1.429.0",
|
||||
@@ -12605,9 +12605,9 @@
|
||||
"integrity": "sha512-EY3mSMWpqFPzd7fsLg2/hSfQFU8HpW9nplFwm4JHHCDbcTpBzlvzjPJoHAAGO5kMzowAxjqi5ai/mXjeUWuiSg=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-regex": {
|
||||
"version": "1.439.0",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.439.0.tgz",
|
||||
"integrity": "sha512-v7vcEOWurGbqvoTdtQ8wauyUYeuQExRCmr7phPtwUwD+1cNbqDrS11kM8p2Na3DXjL9RqGAGjd6uRjNSccZjjQ=="
|
||||
"version": "1.478.2",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.478.2.tgz",
|
||||
"integrity": "sha512-XXYOzCePuHBEdiTREF+UlRGDyUu5VVXD9RBb5JGiq9YkjKhKek0CkjSt8HhJ0yAu0i/gvWLOO0RboT2C04Qt1A=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-rust": {
|
||||
"version": "1.429.0",
|
||||
|
||||
@@ -147,7 +147,7 @@
|
||||
"windmill-parser-wasm-nu": "^1.474.1",
|
||||
"windmill-parser-wasm-php": "^1.429.0",
|
||||
"windmill-parser-wasm-py": "^1.477.1",
|
||||
"windmill-parser-wasm-regex": "^1.439.0",
|
||||
"windmill-parser-wasm-regex": "^1.478.2",
|
||||
"windmill-parser-wasm-rust": "^1.429.0",
|
||||
"windmill-parser-wasm-ts": "^1.429.0",
|
||||
"windmill-parser-wasm-yaml": "^1.429.0",
|
||||
@@ -515,4 +515,4 @@
|
||||
"optionalDependencies": {
|
||||
"fsevents": "^2.3.3"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user