feat: deprecate previous_result in favor of results per id

This commit is contained in:
Ruben Fiszel
2022-11-10 01:58:43 +01:00
parent a218fbf194
commit 40183ce4e4
38 changed files with 547 additions and 527 deletions
+2
View File
@@ -4159,6 +4159,7 @@ version = "1.45.0"
dependencies = [
"anyhow",
"itertools",
"lazy_static",
"phf 0.11.1",
"regex",
"unicode-general-category",
@@ -4184,6 +4185,7 @@ version = "1.45.0"
dependencies = [
"anyhow",
"itertools",
"lazy_static",
"phf 0.11.1",
"regex",
"rustpython-parser",
@@ -15,4 +15,5 @@ phf.workspace = true
unicode-general-category.workspace = true
itertools.workspace = true
anyhow.workspace = true
regex.workspace = true
regex.workspace = true
lazy_static.workspace = true
@@ -17,3 +17,4 @@ itertools.workspace = true
regex.workspace = true
serde_json.workspace = true
anyhow.workspace = true
lazy_static.workspace = true
@@ -9,6 +9,7 @@
use std::collections::HashMap;
use itertools::Itertools;
use lazy_static::lazy_static;
use phf::phf_map;
use regex::Regex;
@@ -189,17 +190,20 @@ fn replace_import(x: String) -> String {
.to_string()
}
lazy_static! {
static ref RE: Regex = Regex::new(r"^\#(\S+)$").unwrap();
}
pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
let find_requirements = code
.lines()
.find_position(|x| x.starts_with("#requirements:"));
let re = Regex::new(r"^\#(\S+)$").unwrap();
if let Some((pos, _)) = find_requirements {
let lines = code
.lines()
.skip(pos + 1)
.map_while(|x| {
re.captures(x)
RE.captures(x)
.map(|x| x.get(1).unwrap().as_str().to_string())
})
.collect();
+21
View File
@@ -2414,6 +2414,27 @@
},
"query": "DELETE FROM token WHERE token = $1 RETURNING email"
},
"a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a": {
"describe": {
"columns": [
{
"name": "result",
"ordinal": 0,
"type_info": "Jsonb"
}
],
"nullable": [
true
],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
}
},
"query": "SELECT result FROM completed_job WHERE id = ANY($1) AND workspace_id = $2"
},
"a241c56415759105ccbcbf7fff77287fa4ec2cc096c0060d14db421115d63e2d": {
"describe": {
"columns": [
+76 -57
View File
@@ -162,6 +162,7 @@ mod suspend_resume {
fn flow() -> FlowValue {
serde_json::from_value(serde_json::json!({
"modules": [{
"id": "a",
"input_transform": {
"n": { "type": "javascript", "expr": "flow_input.n", },
"port": { "type": "javascript", "expr": "flow_input.port", },
@@ -200,8 +201,9 @@ mod suspend_resume {
"required_events": 1
},
}, {
"id": "b",
"input_transform": {
"n": { "type": "javascript", "expr": "previous_result", },
"n": { "type": "javascript", "expr": "results.a", },
"resume": { "type": "javascript", "expr": "resume", },
"resumes": { "type": "javascript", "expr": "resumes", },
},
@@ -215,7 +217,7 @@ mod suspend_resume {
},
}, {
"input_transform": {
"last": { "type": "javascript", "expr": "previous_result", },
"last": { "type": "javascript", "expr": "results.b", },
"resume": { "type": "javascript", "expr": "resume", },
"resumes": { "type": "javascript", "expr": "resumes", },
},
@@ -478,13 +480,14 @@ def main(last, port):
fn flow_forloop_retry() -> FlowValue {
serde_json::from_value(serde_json::json!({
"modules": [{
"id": "a",
"value": {
"type": "forloopflow",
"iterator": { "type": "javascript", "expr": "result.items" },
"iterator": { "type": "javascript", "expr": "flow_input.items" },
"skip_failures": false,
"modules": [{
"input_transform": {
"index": { "type": "javascript", "expr": "previous_result.iter.index" },
"index": { "type": "javascript", "expr": "flow_input.iter.index" },
"port": { "type": "javascript", "expr": "flow_input.port" },
},
"value": {
@@ -497,7 +500,7 @@ def main(last, port):
"retry": { "constant": { "attempts": 2, "seconds": 0 } },
}, {
"input_transform": {
"last": { "type": "javascript", "expr": "previous_result" },
"last": { "type": "javascript", "expr": "results.a" },
"port": { "type": "javascript", "expr": "flow_input.port" },
},
"value": {
@@ -603,14 +606,13 @@ def main(last, port):
.into_iter()
.unzip::<_, _, Vec<_>, Vec<_>>();
let server = Server::start(responses).await;
let result = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None })
let job = RunJob::from(JobPayload::RawFlow { value: flow_forloop_retry(), path: None })
.arg("items", json!(["unused", "unused", "unused"]))
.arg("port", json!(server.addr.port()))
.run_until_complete(&db, server.addr.port())
.await
.result
.unwrap();
.await;
let result = job.result.unwrap();
assert_eq!(server.close().await, attempts);
assert!(result["error"]
.as_str()
@@ -706,7 +708,7 @@ async fn test_iteration(db: Pool<Postgres>) {
"input_transform": {
"n": {
"type": "javascript",
"expr": "previous_result.iter.value",
"expr": "flow_input.iter.value",
},
},
"value": {
@@ -759,7 +761,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
"input_transform": {
"n": {
"type": "javascript",
"expr": "previous_result.iter.value",
"expr": "flow_input.iter.value",
},
},
"value": {
@@ -1027,7 +1029,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
input_transforms: [(
"n".to_string(),
InputTransform::Javascript {
expr: "previous_result.iter.value".to_string(),
expr: "flow_input.iter.value".to_string(),
},
)]
.into(),
@@ -1120,7 +1122,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
(
"i".to_string(),
InputTransform::Javascript {
expr: "previous_result.iter.value".to_string(),
expr: "flow_input.iter.value".to_string(),
},
),
(
@@ -1186,7 +1188,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
input_transforms: [
(
"loops".to_string(),
InputTransform::Javascript { expr: "previous_result".to_string() },
InputTransform::Javascript { expr: "results.b".to_string() },
),
(
"path".to_string(),
@@ -1255,7 +1257,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
"branches": [{"modules": [ {
"id": "d",
"value": {
"input_transforms": {"v": {"type": "javascript", "expr": "result_by_id(\"a\")"}},
"input_transforms": {"v": {"type": "javascript", "expr": "results.a"}},
"type": "rawscript",
"language": "deno",
"content": "export function main(v){ return v }",
@@ -1306,7 +1308,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
{
"id": "b",
"value": {
"input_transforms": { "n": { "type": "javascript", "expr": "previous_result" } },
"input_transforms": { "n": { "type": "javascript", "expr": "results.a" } },
"type": "rawscript",
"language": "python3",
"content": "def main(n): return f'last step saw {n}'",
@@ -1364,7 +1366,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
{
"id": "c",
"value": {
"input_transforms": { "n": { "type": "javascript", "expr": "previous_result" } },
"input_transforms": { "n": { "type": "javascript", "expr": "results.a" } },
"type": "rawscript",
"language": "python3",
"content": "def main(n): return f'last step saw {n}'",
@@ -1425,7 +1427,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
"input_transform": {
"n": {
"type": "javascript",
"expr": "previous_result.iter.value",
"expr": "flow_input.iter.value",
},
},
}],
@@ -1435,7 +1437,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
}))
.unwrap();
for i in 0..50 {
for i in 0..10 {
println!("python flow iteration: {}", i);
let result = run_job_in_new_worker_until_complete(
&db,
@@ -1619,6 +1621,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(serde_json::json!({
"modules": [
{
"id": "a",
"value": {
"type": "forloopflow",
"iterator": { "type": "static", "value": [] },
@@ -1628,7 +1631,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
"input_transform": {
"n": {
"type": "javascript",
"expr": "previous_result.iter.value",
"expr": "flow_input.iter.value",
},
},
"type": "rawscript",
@@ -1644,7 +1647,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
"input_transform": {
"items": {
"type": "javascript",
"expr": "previous_result",
"expr": "results.a",
},
},
"type": "rawscript",
@@ -1721,7 +1724,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
"input_transform": {
"n": {
"type": "javascript",
"expr": "previous_result.iter.value",
"expr": "flow_input.iter.value",
},
},
"value": {
@@ -1754,6 +1757,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(serde_json::json!({
"modules": [
{
"id": "a",
"value": {
"type": "forloopflow",
"iterator": { "type": "static", "value": [2,3,4] },
@@ -1762,7 +1766,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
"input_transform": {
"n": {
"type": "javascript",
"expr": "previous_result.iter.value",
"expr": "flow_input.iter.value",
},
},
"value": {
@@ -1778,7 +1782,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
"input_transform": {
"items": {
"type": "javascript",
"expr": "previous_result",
"expr": "results.a",
},
},
"value": {
@@ -1800,12 +1804,13 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
assert_eq!(result, serde_json::json!(9));
}
fn module_add_item_to_list(i: i32) -> serde_json::Value {
fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value {
json!({
"id": format!("id_{}", i.to_string().replace("-", "_")),
"input_transform": {
"array": {
"type": "javascript",
"expr": "previous_result",
"expr": format!("results.{id}"),
},
"i": {
"type": "static",
@@ -1840,6 +1845,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
@@ -1849,7 +1855,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
{
"value": {
"branches": [],
"default": [module_add_item_to_list(2)],
"default": [module_add_item_to_list(2, "a")],
"type": "branchone",
}
},
@@ -1884,8 +1890,8 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
},
{
"value": {
"branches": [{"expr": "previous_result[0] == 1 && result_by_id(\"a\")[0] == 1", "modules": [module_add_item_to_list(3)]}],
"default": [module_add_item_to_list(2)],
"branches": [{"expr": "results.a[0] == 1", "modules": [module_add_item_to_list(3, "a")]}],
"default": [module_add_item_to_list(2, "a")],
"type": "branchone",
}
},
@@ -1911,6 +1917,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
@@ -1920,8 +1927,8 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
{
"value": {
"branches": [
{"modules": [module_add_item_to_list(2)]},
{"modules": [module_add_item_to_list(3)]}],
{"modules": [module_add_item_to_list(2, "a")]},
{"modules": [module_add_item_to_list(3, "a")]}],
"type": "branchall",
"parallel": true,
}
@@ -1948,6 +1955,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
@@ -1957,8 +1965,8 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
{
"value": {
"branches": [
{"modules": [module_add_item_to_list(2)]},
{"modules": [module_add_item_to_list(3)]}],
{"modules": [module_add_item_to_list(2, "a")]},
{"modules": [module_add_item_to_list(3, "a")]}],
"type": "branchall",
}
},
@@ -1984,6 +1992,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
@@ -1994,7 +2003,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
"value": {
"branches": [
{"modules": [module_failure()], "skip_failure": false},
{"modules": [module_add_item_to_list(3)]}],
{"modules": [module_add_item_to_list(3, "a")]}],
"type": "branchall",
}
},
@@ -2016,6 +2025,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
@@ -2026,7 +2036,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
"value": {
"branches": [
{"modules": [module_failure()], "skip_failure": true},
{"modules": [module_add_item_to_list(2)]}
{"modules": [module_add_item_to_list(2, "a")]}
],
"type": "branchall",
}
@@ -2056,14 +2066,16 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"content": "export function main(){ return [] }",
}
},
module_add_item_to_list(1),
module_add_item_to_list(1, "a"),
{
"id": "b",
"value": {
"branches": [
{
@@ -2079,17 +2091,17 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
"expr": "false",
"modules": []
}],
"default": [module_add_item_to_list(2)],
"default": [module_add_item_to_list(2, "id_1")],
"type": "branchone",
}
}]
},
],
"default": [module_add_item_to_list(-4)],
"default": [module_add_item_to_list(-4, "id_1")],
"type": "branchone",
}
},
module_add_item_to_list(3),
module_add_item_to_list(3, "b"),
],
}))
.unwrap();
@@ -2112,6 +2124,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
@@ -2122,24 +2135,27 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
"value": {
"branches": [
{
"modules": [ {
"modules": [
{
"id": "b",
"value": {
"branches": [
{"modules": [module_add_item_to_list(2)]},
{"modules": [module_add_item_to_list(3)]}],
{"modules": [module_add_item_to_list(2, "a")]},
{"modules": [module_add_item_to_list(3, "a")]}],
"type": "branchall",
}
}, {
"value": {
"branches": [
{"modules": [module_add_item_to_list(4)]},
{"modules": [module_add_item_to_list(5)]}],
{"modules": [module_add_item_to_list(4, "b")]},
{"modules": [module_add_item_to_list(5, "b")]}],
"type": "branchall",
}
}
]
},
{"modules": [module_add_item_to_list(6)]}],
{"modules": [module_add_item_to_list(6, "a")]}],
// "parallel": false,
"type": "branchall",
}
},
@@ -2153,6 +2169,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
.result
.unwrap();
println!("{:#?}", result);
assert_eq!(
result,
serde_json::json!([[[[1, 2], [1, 3], 4], [[1, 2], [1, 3], 5]], [1, 6]])
@@ -2167,31 +2184,33 @@ async fn test_failure_module(db: Pool<Postgres>) {
let flow: FlowValue = serde_json::from_value(serde_json::json!({
"modules": [{
"input_transform": {
"l": { "type": "javascript", "expr": "[]", },
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"id": "a",
"value": {
"input_transform": {
"l": { "type": "javascript", "expr": "[]", },
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"type": "rawscript",
"language": "deno",
"content": "export function main(n, l) { if (n == 0) throw l; return { l: [...l, 0] } }",
},
}, {
"input_transform": {
"l": { "type": "javascript", "expr": "previous_result.l", },
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"id": "b",
"value": {
"input_transform": {
"l": { "type": "javascript", "expr": "results.a.l", },
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"type": "rawscript",
"language": "deno",
"content": "export function main(n, l) { if (n == 1) throw l; return { l: [...l, 1] } }",
},
}, {
"input_transform": {
"l": { "type": "javascript", "expr": "previous_result.l", },
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"value": {
"input_transform": {
"l": { "type": "javascript", "expr": "results.b.l", },
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"type": "rawscript",
"language": "deno",
"content": "export function main(n, l) { if (n == 2) throw l; return { l: [...l, 2] } }",
@@ -117,6 +117,12 @@ pub enum FlowStatusModule {
},
}
#[derive(Debug, Clone)]
pub enum JobResult {
SingleJob(Uuid),
ListJob(Vec<Uuid>),
}
impl FlowStatusModule {
pub fn job(&self) -> Option<Uuid> {
match self {
@@ -129,6 +135,21 @@ impl FlowStatusModule {
}
}
pub fn flow_jobs(&self) -> Option<Vec<Uuid>> {
match self {
FlowStatusModule::InProgress { flow_jobs, .. } => flow_jobs.clone(),
FlowStatusModule::Success { flow_jobs, .. } => flow_jobs.clone(),
FlowStatusModule::Failure { flow_jobs, .. } => flow_jobs.clone(),
_ => None,
}
}
pub fn job_result(&self) -> Option<JobResult> {
self.flow_jobs()
.map(JobResult::ListJob)
.or_else(|| self.job().map(JobResult::SingleJob))
}
pub fn id(&self) -> String {
match self {
FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(),
+30 -12
View File
@@ -16,7 +16,7 @@ use uuid::Uuid;
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{self, to_anyhow, Error},
flow_status::{FlowStatus, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
flows::FlowValue,
scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
utils::StripPath,
@@ -124,7 +124,7 @@ pub async fn get_result_by_id(
flow_id: String,
node_id: String,
) -> error::Result<serde_json::Value> {
let mut result_id: Option<Uuid> = None;
let mut result_id: Option<JobResult> = None;
let mut parent_id = Uuid::from_str(&flow_id).ok();
while result_id.is_none() && parent_id.is_some() {
if !skip_direct {
@@ -148,7 +148,7 @@ pub async fn get_result_by_id(
.modules
.iter()
.find(|m| m.id() == node_id)
.and_then(|m| m.job())
.and_then(|m| m.job_result())
});
} else {
parent_id = None;
@@ -171,15 +171,33 @@ pub async fn get_result_by_id(
"Flow result by id",
format!("{}, {}", flow_id, node_id),
)?;
let value = sqlx::query_scalar!(
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
result_id,
w_id,
)
.fetch_optional(&db)
.await?
.flatten()
.unwrap_or(serde_json::Value::Null);
println!("result_id: {:#?}, {node_id}", result_id);
let value = match result_id {
JobResult::ListJob(x) => {
let rows = sqlx::query_scalar!(
"SELECT result FROM completed_job WHERE id = ANY($1) AND workspace_id = $2",
x.as_slice(),
w_id,
)
.fetch_all(&db)
.await?
.into_iter()
.filter_map(|x| x)
.collect::<Vec<serde_json::Value>>();
serde_json::json!(rows)
}
JobResult::SingleJob(x) => sqlx::query_scalar!(
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
x,
w_id,
)
.fetch_optional(&db)
.await?
.flatten()
.unwrap_or(serde_json::Value::Null),
};
Ok(value)
}
+1 -1
View File
@@ -44,4 +44,4 @@ lazy_static.workspace = true
chrono.workspace = true
dotenv.workspace = true
rand.workspace = true # TODO: Remove. only used by token creation hack.
deno_core.workspace = true
deno_core.workspace = true
+56 -44
View File
@@ -10,11 +10,12 @@ use std::collections::HashMap;
use deno_core::{op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, RuntimeOptions};
use itertools::Itertools;
use lazy_static::lazy_static;
use regex::Regex;
use serde_json::Value;
use tokio::{sync::oneshot, time::timeout};
use uuid::Uuid;
use windmill_common::error::Error;
use windmill_common::{error::Error, flow_status::JobResult};
pub struct EvalCreds {
pub workspace: String,
@@ -22,13 +23,16 @@ pub struct EvalCreds {
}
#[derive(Debug, Clone)]
pub struct IdContext(pub Uuid, pub HashMap<String, Uuid>);
pub struct IdContext {
pub flow_job: Uuid,
pub steps_results: HashMap<String, JobResult>,
pub previous_id: String,
}
pub async fn eval_timeout(
expr: String,
env: Vec<(String, serde_json::Value)>,
creds: Option<EvalCreds>,
steps: Vec<Uuid>,
by_id: Option<IdContext>,
base_internal_url: String,
) -> anyhow::Result<serde_json::Value> {
@@ -49,12 +53,9 @@ pub async fn eval_timeout(
])
}
if !steps.is_empty() || by_id.is_some() {
ops.push(op_get_result::decl())
}
if by_id.is_some() {
ops.push(op_get_id::decl())
ops.push(op_get_result::decl());
ops.push(op_get_id::decl());
}
let ext = Extension::builder().ops(ops).build();
@@ -82,12 +83,13 @@ pub async fn eval_timeout(
.into_iter()
.fold(expr, replace_with_await);
let expr = replace_with_await_result(expr);
let r = runtime.block_on(eval(
&mut js_runtime,
&expr,
env,
creds,
steps,
by_id,
&base_internal_url,
))?;
@@ -115,6 +117,13 @@ fn replace_with_await(expr: String, fn_name: &str) -> String {
}
s
}
lazy_static! {
static ref RE: Regex = Regex::new("(?m)(?P<r>results.([a-z]|[A-Z]|_|[1-9])+)").unwrap();
}
fn replace_with_await_result(expr: String) -> String {
RE.replace_all(&expr, "(await $r)").to_string()
}
fn add_closing_bracket(s: &str) -> String {
let mut s = s.to_string();
@@ -141,7 +150,6 @@ async fn eval(
expr: &str,
env: Vec<(String, serde_json::Value)>,
creds: Option<EvalCreds>,
steps: Vec<Uuid>,
by_id: Option<IdContext>,
base_internal_url: &str,
) -> anyhow::Result<serde_json::Value> {
@@ -153,47 +161,52 @@ async fn eval(
.join("\n"),
expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
);
let (steps_code, api_code, by_id_code) = if let Some(EvalCreds { workspace, token }) = creds {
let steps_code = if !steps.is_empty() {
format!(
r#"
let steps = [{}];
async function step(n) {{
if (n == -1) {{
return previous_result;
}}
if (n < 0) {{
let steps_length = steps.length;
n = n % steps.length + steps.length;
}}
let id = steps[n];
return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
}}"#,
steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
)
} else {
String::new()
};
let (api_code, by_id_code) = if let Some(EvalCreds { workspace, token }) = creds {
let by_id_code = if let Some(by_id) = by_id {
format!(
r#"
async function result_by_id(node_id) {{
let id_map = {{ {} }};
let id = id_map[node_id];
if (id) {{
return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
if (node_id == "{}") {{
return previous_result;
}} else if (id) {{
if (Array.isArray(id)) {{
return await Promise.all(id.map(async (id) => await get_result(id)));
}} else {{
return await get_result(id);
}}
}} else {{
let flow_job_id = "{}";
return await Deno.core.opAsync("op_get_id", [workspace, flow_job_id, token, base_url, node_id]);
}}
}}"#,
}}
async function get_result(id) {{
return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
}}
const results = new Proxy({{}}, {{
get: function(target, name, receiver) {{
return result_by_id(name);
}}
}});
"#,
by_id
.1
.steps_results
.into_iter()
.map(|(k, v)| format!("\"{k}\": \"{v}\""))
.map(|(k, v)| {
let v_str = match v {
JobResult::SingleJob(x) => x.to_string(),
JobResult::ListJob(x) => {
format!("[{}]", x.iter().map(|x| x.to_string()).join(","))
}
};
format!("\"{k}\": \"{v_str}\"")
})
.join(","),
by_id.0,
by_id.previous_id,
by_id.flow_job,
)
} else {
String::new()
@@ -213,16 +226,15 @@ async function resource(path) {{
"#,
base_internal_url,
);
(steps_code, api_code, by_id_code)
(api_code, by_id_code)
} else {
(String::new(), String::new(), String::new())
(String::new(), String::new())
};
let code = format!(
r#"
{api_code}
{}
{steps_code}
{by_id_code}
(async () => {{
{expr}
@@ -334,7 +346,7 @@ mod tests {
let code = "value.test + params.test";
let mut runtime = JsRuntime::new(RuntimeOptions::default());
let res = eval(&mut runtime, code, env, None, vec![], None, "").await?;
let res = eval(&mut runtime, code, env, None, None, "").await?;
assert_eq!(res, json!(4));
Ok(())
}
@@ -347,7 +359,7 @@ mod tests {
multiline template`";
let mut runtime = JsRuntime::new(RuntimeOptions::default());
let res = eval(&mut runtime, code, env, None, vec![], None, "").await?;
let res = eval(&mut runtime, code, env, None, None, "").await?;
assert_eq!(res, json!("my 5\nmultiline template"));
Ok(())
}
@@ -360,7 +372,7 @@ multiline template`";
];
let code = r#"params.test"#;
let res = eval_timeout(code.to_string(), env, None, vec![], None, "".to_string()).await?;
let res = eval_timeout(code.to_string(), env, None, None, "".to_string()).await?;
assert_eq!(res, json!(2));
Ok(())
}
+27 -65
View File
@@ -18,7 +18,7 @@ use serde_json::{json, Map, Value};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use uuid::Uuid;
use windmill_common::flow_status::Iterator;
use windmill_common::flow_status::{Iterator, JobResult};
use windmill_common::{
error::{self, to_anyhow, Error},
flow_status::{
@@ -77,6 +77,7 @@ pub async fn update_flow_status_after_job_completion(
})?;
let module_index = usize::try_from(old_status.step).ok();
let module_status = module_index
.and_then(|i| old_status.modules.get(i))
.unwrap_or(&old_status.failure_module);
@@ -578,7 +579,6 @@ async fn compute_bool_from_expr(
]
.into(),
creds,
vec![],
by_id,
base_internal_url.to_string(),
)
@@ -657,7 +657,6 @@ async fn transform_input(
input_transforms: &HashMap<String, InputTransform>,
workspace: &str,
token: &str,
steps: Vec<Uuid>,
resumes: &[Value],
approvers: Vec<String>,
by_id: &IdContext,
@@ -676,7 +675,7 @@ async fn transform_input(
InputTransform::Static { value: _ } => (),
InputTransform::Javascript { expr } => {
let flow_input = flow_args.clone().unwrap_or_else(|| json!({}));
let previous_result = flatten_previous_result(last_result.clone());
let previous_result = last_result.clone();
let context = vec![
("params".to_string(), json!(mapped)),
("previous_result".to_string(), previous_result),
@@ -693,7 +692,6 @@ async fn transform_input(
expr.to_string(),
context,
Some(EvalCreds { workspace: workspace.to_string(), token: token.to_string() }),
steps.clone(),
Some(by_id.clone()),
base_internal_url.to_string(),
)
@@ -712,24 +710,6 @@ async fn transform_input(
Ok(mapped)
}
fn flatten_previous_result(last_result: serde_json::Value) -> serde_json::Value {
if last_result.is_object()
&& last_result
.as_object()
.unwrap()
.contains_key("previous_result")
{
last_result
.as_object()
.unwrap()
.get("previous_result")
.unwrap()
.clone()
} else {
last_result.clone()
}
}
#[instrument(level = "trace", skip_all)]
pub async fn handle_flow(
flow_job: &QueuedJob,
@@ -820,6 +800,12 @@ async fn push_next_flow_job(
.or_else(|| flow.failure_module.as_ref())
.with_context(|| format!("no module at index {}", status.step))?;
let previous_id = if i >= 1 {
flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap()
} else {
String::new()
};
// calculate sleep if any
let mut scheduled_for_o = {
let sleep_input_transform = i
@@ -834,7 +820,6 @@ async fn push_next_flow_job(
expr.to_string(),
[("result".to_string(), last_result.clone())].into(),
None,
vec![],
None,
"".to_string(),
)
@@ -1083,9 +1068,9 @@ async fn push_next_flow_job(
let mut args = match &module.value {
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. } => {
let ctx = get_transform_context(db, &flow_job, &status, &flow.modules).await?;
let ctx = get_transform_context(db, &flow_job, previous_id.clone(), &status).await?;
transform_context = Some(ctx);
let (token, steps, by_id) = transform_context.as_ref().unwrap();
let (token, by_id) = transform_context.as_ref().unwrap();
transform_input(
&flow_job.args,
last_result.clone(),
@@ -1096,7 +1081,6 @@ async fn push_next_flow_job(
},
&flow_job.workspace_id,
&token,
steps.to_vec(),
resume_messages.as_slice(),
approvers,
by_id,
@@ -1140,6 +1124,7 @@ async fn push_next_flow_job(
&status_module,
last_result.clone(),
base_internal_url,
previous_id,
)
.await?;
tx.commit().await?;
@@ -1177,31 +1162,12 @@ async fn push_next_flow_job(
args.extend(new_args.clone());
vec![args]
}
NextStatus::BranchChosen(_) => {
args.insert(
"previous_result".to_string(),
flatten_previous_result(last_result),
);
vec![args]
}
NextStatus::NextBranchStep(NextBranch { status, .. }) => {
args.insert(
"previous_result".to_string(),
flatten_previous_result(status.previous_result.clone()),
);
vec![args]
}
NextStatus::AllFlowJobs {
branchall: Some(BranchAllStatus { len, .. }),
iterator: None,
..
} => {
args.insert(
"previous_result".to_string(),
flatten_previous_result(last_result),
);
(0..*len).map(|_| args.clone()).collect()
}
} => (0..*len).map(|_| args.clone()).collect(),
NextStatus::AllFlowJobs {
branchall: None,
iterator: Some(Iterator { itered, .. }),
@@ -1459,7 +1425,7 @@ async fn script_path_to_payload<'c>(
Ok(job_payload)
}
type TransformContext = (String, Vec<Uuid>, IdContext);
type TransformContext = (String, IdContext);
async fn compute_next_flow_transform<'c>(
flow_job: &QueuedJob,
@@ -1472,6 +1438,7 @@ async fn compute_next_flow_transform<'c>(
status_module: &FlowStatusModule,
last_result: serde_json::Value,
base_internal_url: &str,
previous_id: String,
) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, NextFlowTransform)> {
match &module.value {
FlowModuleValue::Identity => Ok((
@@ -1508,10 +1475,10 @@ async fn compute_next_flow_transform<'c>(
let next_loop_status = match status_module {
FlowStatusModule::WaitingForPriorSteps { .. } => {
let (token, steps, by_id) = if let Some(x) = transform_context {
let (token, by_id) = if let Some(x) = transform_context {
x
} else {
get_transform_context(db, &flow_job, &status, &flow.modules).await?
get_transform_context(db, &flow_job, previous_id, &status).await?
};
let flow_input = flow_job.args.clone().unwrap_or_else(|| json!({}));
/* Iterator is an InputTransform, evaluate it into an array. */
@@ -1526,7 +1493,6 @@ async fn compute_next_flow_transform<'c>(
},
token,
flow_job.workspace_id.clone(),
steps,
Some(by_id),
base_internal_url,
)
@@ -1629,8 +1595,8 @@ async fn compute_next_flow_transform<'c>(
let branch = match status_module {
FlowStatusModule::WaitingForPriorSteps { .. } => {
let mut branch_chosen = BranchChosen::Default;
let (token, _steps, idcontext) =
get_transform_context(db, &flow_job, &status, &flow.modules).await?;
let (token, idcontext) =
get_transform_context(db, &flow_job, previous_id, &status).await?;
for (i, b) in branches.iter().enumerate() {
let pred = compute_bool_from_expr(
b.expr.to_string(),
@@ -1787,8 +1753,8 @@ async fn compute_next_flow_transform<'c>(
async fn get_transform_context(
db: &DB,
flow_job: &QueuedJob,
previous_id: String,
status: &FlowStatus,
modules: &Vec<FlowModule>,
) -> error::Result<TransformContext> {
let tx = db.begin().await?;
let (tx, new_token) = crate::create_token_for_owner(
@@ -1803,18 +1769,16 @@ async fn get_transform_context(
//we need to commit asap otherwise the token won't be valid for auth to check outside of this transaction
//which will happen with client http calls
tx.commit().await?;
let new_steps: Vec<Uuid> = status
let steps_results: HashMap<String, JobResult> = status
.modules
.iter()
.map(|x| x.job().unwrap_or_default())
.collect();
let id_map: HashMap<String, Uuid> = modules
.iter()
.map(|x| x.id.clone())
.zip(new_steps.clone())
.filter_map(|x| x.job_result().map(|y| (x.id(), y)))
.collect();
Ok((new_token, new_steps, IdContext(flow_job.id, id_map)))
Ok((
new_token,
IdContext { flow_job: flow_job.id, steps_results, previous_id },
))
}
async fn evaluate_with<F>(
@@ -1822,7 +1786,6 @@ async fn evaluate_with<F>(
vars: F,
token: String,
workspace: String,
steps: Vec<Uuid>,
by_id: Option<IdContext>,
base_internal_url: &str,
) -> anyhow::Result<serde_json::Value>
@@ -1836,7 +1799,6 @@ where
expr,
vars(),
Some(EvalCreds { workspace, token }),
steps,
by_id,
base_internal_url.to_string(),
)
@@ -346,7 +346,6 @@
placeholder={defaultValue ?? ''}
bind:value
on:input={() => {
console.log(6, value)
dispatch('input', { rawValue: value, isRaw: false })
}}
/>
@@ -5,7 +5,7 @@
import { Button } from './common'
import { createEventDispatcher, getContext } from 'svelte'
import Icon from 'svelte-awesome'
import { dfs, flowIds, flowStore } from './flows/flowStore'
import { dfs, flowStore } from './flows/flowStore'
import type { FlowEditorContext } from './flows/types'
import { runFlowPreview } from './flows/utils'
import SchemaForm from './SchemaForm.svelte'
@@ -53,7 +53,7 @@
return $flowStore
} else {
const flow: Flow = JSON.parse(JSON.stringify($flowStore))
const idOrders = dfs(flow.value.modules, true)
const idOrders = dfs(flow.value.modules)
let upToIndex = idOrders.indexOf($selectedId)
if (upToIndex != -1) {
@@ -52,7 +52,7 @@
{/if}
<p class="font-black text-lg w-full my-4">
<span>Flow Inputs</span>
<span>Flow Input</span>
</p>
{#if flow.schema && flow.schema.properties && Object.keys(flow.schema.properties).length > 0 && flow.schema}
<ul class="my-2">
@@ -94,9 +94,6 @@
job = await JobService.getJob({ workspace: workspace!, id })
}
} else {
console.log(workspaceOverride)
console.log(workspace)
job = await JobService.getJob({ workspace: workspace!, id })
}
if (job?.type === 'CompletedJob') {
@@ -35,10 +35,12 @@
That snippet can be a single line:
<pre><code>last_result.myarg</code></pre>
or a multiline:
<pre><code
<pre
><code
>let x = 5;
x + 2</code
></pre>
></pre
>
<p>
If it is multiline, the statement before the final expression <b
>MUST END WITH ; and a newline</b
@@ -74,7 +76,7 @@ x + 2</code
</ul>
<p>To re-enable editor assistance, import the helper functions types using:</p>
<code>
{`import { previous_result, flow_input, step, variable, resource, params } from 'windmill${
{`import { results, flow_input, variable, resource, params } from 'windmill${
importPath ? `@${importPath}` : ''
}'`}
</code>
@@ -16,7 +16,7 @@
expr: string
modules: Array<FlowModule>
}
export let parentModule: FlowModule | undefined
export let parentModule: FlowModule
export let previousModule: FlowModule | undefined
const { previewArgs } = getContext<FlowEditorContext>('FlowEditorContext')
@@ -26,8 +26,10 @@
$flowStateStore,
parentModule,
previousModule,
parentModule.id,
$flowStore,
previewArgs,
false,
true
).pickableProperties
</script>
@@ -44,7 +46,7 @@
<span class="mb-2 text-sm font-bold">Branch predicate</span>
<div class="border w-full">
<PropPickerWrapper
priorId={previousModule?.id}
notSelectable
{pickableProperties}
on:select={({ detail }) => {
editor?.insertAtCursor(detail)
@@ -127,11 +127,7 @@
<TabContent value="early-stop" class="flex flex-col flex-1 h-full">
<div class="p-4 overflow-y-auto">
<FlowModuleEarlyStop
previousModuleId={previousModule?.id}
bind:flowModule
{parentModule}
/>
<FlowModuleEarlyStop bind:flowModule />
</div>
</TabContent>
@@ -12,7 +12,7 @@
</script>
<CapturePayload bind:this={capturePayload} />
<FlowCard title="Flow Inputs">
<FlowCard title="Flow Input">
<div slot="header">
<div class="flex flex-row space-x-4">
<Button
@@ -15,7 +15,7 @@
<div class="space-y-4 p-4">
<div class="text-sm font-bold">{failureModule ? 'Error handler' : 'Common script'}</div>
<div class="grid sm:grid-col-2 lg:grid-cols-3 gap-4">
<div class="grid sm:grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
<FlowScriptPicker
label="Inline Python (3.10)"
icon={faCode}
@@ -100,7 +100,7 @@
</Tooltip>
</div>
<div class="grid sm:grid-col-1 md:grid-col-2 lg:grid-cols-3 gap-4">
<div class="grid sm:grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
<PickScript customText="Trigger script from workspace" kind={Script.kind.TRIGGER} on:pick />
<PickHubScript customText="Trigger script from Hub" kind={Script.kind.TRIGGER} on:pick />
<FlowScriptPicker
@@ -121,7 +121,7 @@
Use getResumeEndpoints from the wmill client to generate those URLs.
</Tooltip>
</div>
<div class="grid sm:grid-col-1 md:grid-col-2 lg:grid-cols-3 gap-4">
<div class="grid sm:grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
<PickScript customText="Approval step from workspace" kind={Script.kind.APPROVAL} on:pick />
<PickHubScript
customText={'Approval step from the Hub'}
@@ -138,7 +138,7 @@
<div class="text-sm font-bold pt-8">Flow primitive</div>
<div class="grid sm:grid-col-1 md:grid-col-2 lg:grid-cols-3 gap-4">
<div class="grid sm:grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
<FlowScriptPicker
label={`Branches and switch to one`}
icon={faCodeBranch}
@@ -29,8 +29,10 @@
$flowStateStore,
parentModule,
previousModule,
mod.id,
$flowStore,
previewArgs,
false,
true
).pickableProperties
</script>
@@ -53,7 +55,7 @@
{#if mod.value.iterator.type == 'javascript'}
<div class="border w-full">
<PropPickerWrapper
priorId={previousModule?.id}
notSelectable
{pickableProperties}
on:select={({ detail }) => {
editor?.insertAtCursor(detail)
@@ -107,11 +109,7 @@
<TabContent value="early-stop" class="flex flex-col flex-1 h-full">
<div class="p-4 overflow-y-auto">
<FlowModuleEarlyStop
previousModuleId={previousModule?.id}
bind:flowModule={mod}
{parentModule}
/>
<FlowModuleEarlyStop bind:flowModule={mod} />
</div>
</TabContent>
@@ -55,13 +55,22 @@
}
$: stepPropPicker = failureModule
? { pickableProperties: { previous_result: { error: 'the error message' } }, extraLib: '' }
? {
pickableProperties: {
flow_input: $flowStateStore.previewArgs,
priorIds: {},
previousId: undefined
},
extraLib: ''
}
: getStepPropPicker(
$flowStateStore,
parentModule,
previousModule,
flowModule.id,
$flowStore,
previewArgs,
false,
true
)
@@ -183,7 +192,7 @@
><Tooltip>
Move the focus outside of the text editor to recompute the inputs or press
<Kbd>Ctrl/Cmd</Kbd> + <Kbd>S</Kbd>
</Tooltip><span class="font-semibold">Step Inputs</span></Tab
</Tooltip><span class="font-semibold">Step Input</span></Tab
>
<Tab value="test"><span class="font-semibold text-md">Test this step</span></Tab>
<Tab value="retries">Retries</Tab>
@@ -196,10 +205,7 @@
<div class="h-[calc(100%-32px)]">
{#if selected === 'inputs'}
<div class="h-full overflow-auto">
<PropPickerWrapper
priorId={previousModule?.id}
pickableProperties={stepPropPicker.pickableProperties}
>
<PropPickerWrapper pickableProperties={stepPropPicker.pickableProperties}>
<SchemaForm
schema={$flowStateStore[$selectedId]?.schema ?? {}}
inputTransform={true}
@@ -218,12 +224,7 @@
{:else if selected === 'retries'}
<FlowRetries bind:flowModule class="px-4 pb-4 h-full overflow-auto" />
{:else if selected === 'early-stop'}
<FlowModuleEarlyStop
previousModuleId={previousModule?.id}
bind:flowModule
class="px-4 pb-4 h-full overflow-auto"
{parentModule}
/>
<FlowModuleEarlyStop bind:flowModule class="px-4 pb-4 h-full overflow-auto" />
{:else if selected === 'suspend'}
<div class="px-4 pb-4 h-full overflow-auto">
<FlowModuleSuspend previousModuleId={previousModule?.id} bind:flowModule />
@@ -3,37 +3,17 @@
import Toggle from '$lib/components/Toggle.svelte'
import PropPickerWrapper from '$lib/components/flows/propPicker/PropPickerWrapper.svelte'
import type { FlowModule } from '$lib/gen'
import type { FlowEditorContext } from '../types'
import { getContext } from 'svelte'
import { getStepPropPicker } from '../previousResults'
import { flowStateStore } from '../flowState'
import { flowStore } from '../flowStore'
import Tooltip from '$lib/components/Tooltip.svelte'
const { previewArgs } = getContext<FlowEditorContext>('FlowEditorContext')
import { NEVER_TESTED_THIS_FAR } from '../utils'
export let flowModule: FlowModule
export let parentModule: FlowModule | undefined
export let previousModuleId: string | undefined
let editor: SimpleEditor | undefined = undefined
$: isStopAfterIfEnabled = Boolean(flowModule.stop_after_if)
let pickableProperties: Record<string, any> = {}
$: {
const propPicker = getStepPropPicker(
$flowStateStore,
parentModule,
flowModule,
$flowStore,
previewArgs
).pickableProperties
propPicker['result'] = propPicker['previous_result']
delete propPicker['previous_result']
pickableProperties = propPicker
}
$: result = $flowStateStore[flowModule.id]?.previewResult ?? NEVER_TESTED_THIS_FAR
</script>
<div class="flex flex-col items-start space-y-2 {$$props.class}">
@@ -61,7 +41,7 @@
}}
/>
<div class="border p-2 flex flex-col {flowModule.stop_after_if ? '' : 'bg-gray-50'}">
<div class="w-full border p-2 flex flex-col {flowModule.stop_after_if ? '' : 'bg-gray-50'}">
{#if flowModule.stop_after_if}
<Toggle
bind:checked={flowModule.stop_after_if.skip_if_stopped}
@@ -72,8 +52,8 @@
<span class="text-xs font-bold">Stop condition expression</span>
<div class="border w-full">
<PropPickerWrapper
priorId={previousModuleId}
{pickableProperties}
{result}
pickableProperties={undefined}
on:select={({ detail }) => {
editor?.insertAtCursor(detail)
}}
@@ -24,9 +24,7 @@
let editor: SimpleEditor | undefined = undefined
const pickableProperties = {
result: $flowStateStore[$selectedId]?.previewResult ?? {}
}
const result = $flowStateStore[$selectedId]?.previewResult ?? {}
$: isSuspendEnabled = Boolean(flowModule.suspend)
$: isSleepEnabled = Boolean(flowModule.sleep)
@@ -101,9 +99,10 @@
{#if flowModule.sleep && schema.properties['sleep']}
<div class="border">
<PropPickerWrapper
priorId={previousModuleId}
notSelectable
{result}
displayContext={false}
{pickableProperties}
pickableProperties={undefined}
on:select={({ detail }) => {
editor?.insertAtCursor(detail)
}}
@@ -117,7 +117,7 @@
{/each}
{:else if flowModule.value.type === 'branchone'}
{#if $selectedId === `${flowModule?.id}-branch-default`}
<div class="p-4 text-sm">Default branch</div>
<div class="p-4 text-sm truncate">Default branch</div>
{:else}
{#each flowModule.value.default as submodule, index}
<svelte:self
+4 -20
View File
@@ -16,33 +16,18 @@ export const flowStore = writable<Flow>({
extra_perms: {}
})
export function dfs(modules: FlowModule[], previewOrder: boolean = false): string[] {
export function dfs(modules: FlowModule[]): string[] {
let result: string[] = []
for (const module of modules) {
if (module.value.type == 'forloopflow') {
if (previewOrder) {
result = result.concat(module.id)
}
result = result.concat(module.id)
result = result.concat(dfs(module.value.modules))
if (!previewOrder) {
result = result.concat(module.id)
}
} else if (module.value.type == 'branchone') {
if (previewOrder) {
result = result.concat(module.id)
}
result = result.concat(module.id)
result = result.concat(dfs(module.value.branches.map((b) => b.modules).flat().concat(module.value.default)))
if (!previewOrder) {
result = result.concat(module.id)
}
} else if (module.value.type == 'branchall') {
if (previewOrder) {
result = result.concat(module.id)
}
result = result.concat(module.id)
result = result.concat(dfs(module.value.branches.map((b) => b.modules).flat()))
if (!previewOrder) {
result = result.concat(module.id)
}
} else {
result.push(module.id)
}
@@ -50,7 +35,6 @@ export function dfs(modules: FlowModule[], previewOrder: boolean = false): strin
return result
}
export const flowIds = derived(flowStore, flow => dfs(flow.value.modules))
export async function initFlow(flow: Flow) {
@@ -48,7 +48,7 @@
{#if module.value.type === 'branchall'}
<div class="flex text-xs">
<div
class="w-full space-y-2 flex flex-col border p-2 bg-gray-500 bg-opacity-10 rounded-sm my-2"
class="w-full space-y-2 flex flex-col border p-2 bg-gray-500 border-gray-600 bg-opacity-10 rounded-sm my-2"
>
{#each module.value.branches ?? [] as branch, branchIndex (branchIndex)}
<!-- svelte-ignore a11y-click-events-have-key-events -->
@@ -65,7 +65,9 @@
)}
>
<Icon data={faCodeBranch} class="mr-2" />
<span class="text-xs flex flex-row justify-between w-full flex-wrap gap-2 items-center">
<span
class="text-xs flex flex-row justify-between w-full flex-wrap gap-2 items-center truncate"
>
{branch.summary || `Branch ${branchIndex}`}
<Button
iconOnly
@@ -82,7 +84,7 @@
<FlowModuleSchemaMap bind:modules={branch.modules} color="indigo" />
</div>
{/each}
<div>
<div class="overflow-clip">
<Button
size="xs"
color="dark"
@@ -49,7 +49,7 @@
{#if module.value.type === 'branchone'}
<div class="flex text-xs">
<div
class="w-full space-y-2 flex flex-col border p-2 bg-gray-500 bg-opacity-10 rounded-sm my-2"
class="w-full space-y-2 flex flex-col border p-2 bg-gray-500 border-gray-600 bg-opacity-10 rounded-sm my-2"
>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div
@@ -65,15 +65,15 @@
)}
>
<Icon data={faCodeBranch} class="mr-2" />
<span class="text-xs flex flex-row justify-between w-full flex-wrap gap-2 items-center">
<span
class="truncate text-xs flex flex-row justify-between w-full flex-wrap gap-2 items-center"
>
Default branch
</span>
</div>
{#if selectedBranch === 0}
<div transition:slide>
<FlowModuleSchemaMap bind:modules={module.value.default} color="indigo" />
</div>
{/if}
<div>
<FlowModuleSchemaMap bind:modules={module.value.default} color="indigo" />
</div>
{#each module.value.branches ?? [] as branch, branchIndex (branchIndex)}
<!-- svelte-ignore a11y-click-events-have-key-events -->
@@ -90,7 +90,9 @@
)}
>
<Icon data={faCodeBranch} class="mr-2" />
<span class="text-xs flex flex-row justify-between w-full flex-wrap gap-2 items-center">
<span
class="text-xs flex flex-row justify-between w-full flex-wrap gap-2 items-center truncate"
>
{branch.summary || `Branch ${branchIndex}`}
<Button
@@ -108,7 +110,7 @@
<FlowModuleSchemaMap bind:modules={branch.modules} color="indigo" />
</div>
{/each}
<div>
<div class="overflow-clip">
<Button
btnClasses=""
size="xs"
@@ -14,7 +14,7 @@
hasLine
selected={$selectedId === 'inputs'}
bold
label="Flow Inputs"
label="Flow Input"
>
<div slot="icon">
<Icon data={faPen} scale={0.8} />
@@ -1,14 +1,12 @@
import type { Schema } from '$lib/common'
import type { Flow, FlowModule, Job } from '$lib/gen'
import { buildExtraLib, objectToTsType, schemaToObject } from '$lib/utils'
import { schemaToObject } from '$lib/utils'
import type { FlowState } from './flowState'
type Result = any
type PickableProperties = {
flow_input?: Object
previous_result: Result | undefined
step?: Result[]
export type PickableProperties = {
flow_input: Object
priorIds: Record<string, any>
previousId: string | undefined
}
type StepPropPicker = {
@@ -16,13 +14,10 @@ type StepPropPicker = {
extraLib: string
}
type ParentModule = {
parentModule: FlowModule
parentPreviousModuleId: string | undefined
}
type ModuleBranches = FlowModule[][]
function dfs(id: string | undefined, flow: Flow): ParentModule[] {
function dfs(id: string | undefined, flow: Flow, getParents: boolean = true): FlowModule[] {
if (id === undefined) {
return []
}
@@ -40,25 +35,24 @@ function dfs(id: string | undefined, flow: Flow): ParentModule[] {
return []
}
function rec(id: string, moduleBranches: ModuleBranches): ParentModule[] | undefined {
function rec(id: string, moduleBranches: ModuleBranches): FlowModule[] | undefined {
for (let modules of moduleBranches) {
let parentPreviousModuleId: string | undefined = undefined
for (let module of modules) {
for (const [i, module] of modules.entries()) {
if (module.id === id) {
return [{ parentModule: module, parentPreviousModuleId }]
return getParents ? [module] : modules.slice(0, i + 1).reverse()
} else {
const submodules = getSubModules(module)
if (submodules) {
let found: ParentModule[] | undefined = rec(id, submodules)
let found: FlowModule[] | undefined = rec(id, submodules)
if (module && found) {
return [...found, { parentModule: module, parentPreviousModuleId }]
if (found) {
return getParents ? [...found, module] : [...found, ...modules.slice(0, i).reverse()]
}
}
}
parentPreviousModuleId = module.id
}
}
return undefined
@@ -67,24 +61,13 @@ function dfs(id: string | undefined, flow: Flow): ParentModule[] {
return rec(id, [flow.value.modules]) ?? []
}
function flattenPreviousResult(pr: any) {
if (typeof pr === 'object' && pr.previous_result) {
return pr.previous_result
}
return pr
}
function getFlowInput(
parentModules: ParentModule[],
parentModules: FlowModule[],
flowState: FlowState,
args: any,
schema: Schema
) {
const { parentModule, parentPreviousModuleId } = parentModules.shift() ?? {
parentModule: undefined,
parentPreviousModuleId: undefined
}
const parentModule = parentModules.shift()
const parentState = parentModule ? flowState[parentModule.id] : undefined
@@ -103,16 +86,7 @@ function getFlowInput(
...parentFlowInput,
}
} else {
// Branches
if (parentPreviousModuleId === undefined) {
return parentFlowInput
} else {
return {
...parentFlowInput,
previous_result: flattenPreviousResult(flowState[parentPreviousModuleId]?.previewResult ?? {})
}
}
return parentFlowInput
}
}
} else {
@@ -124,26 +98,67 @@ export function getStepPropPicker(
flowState: FlowState,
parentModule: FlowModule | undefined,
previousModule: FlowModule | undefined,
id: string,
flow: Flow,
args: any,
include_node: boolean,
approvers: boolean = false
): StepPropPicker {
const flowInput = getFlowInput(dfs(parentModule?.id, flow), flowState, args, flow.schema)
const previousResults = previousModule
? flowState[previousModule.id]?.previewResult
: flattenPreviousResult(flowInput)
const previousIds = dfs(id, flow, false).map((x) => x.id)
if (!include_node) {
previousIds.shift()
}
let priorIds = Object.fromEntries(previousIds.map((id) => [id, flowState[id]?.previewResult ?? {}]))
const pickableProperties = {
flow_input: flowInput,
previous_result: previousResults
priorIds,
previousId: previousIds[0]
}
if (approvers && ((previousModule?.suspend?.required_events ?? 0) > 0)) {
pickableProperties["approvers"] = "The list of approvers"
}
return {
extraLib: buildExtraLib(objectToTsType(flowInput), objectToTsType(previousResults)),
extraLib: buildExtraLib(flowInput, priorIds),
pickableProperties
}
}
export function buildExtraLib(flowInput: Record<string, any>, results: Record<string, any>): string {
return `
/**
* get variable (including secret) at path
* @param {string} path - path of the variable (e.g: g/all/pretty_secret)
*/
export function variable(path: string): string;
/**
* get resource at path
* @param {string} path - path of the resource (e.g: g/all/my_resource)
*/
export function resource(path: string): any;
/**
* flow input as an object
*/
export const flow_input = ${JSON.stringify(flowInput)};
/**
* static params of this same step
*/
export const params: any;
/**
* result by id
*/
export const results = ${JSON.stringify(results)};
`
}
@@ -18,13 +18,17 @@
<script lang="ts">
import PropPicker from '$lib/components/propertyPicker/PropPicker.svelte'
import PropPickerResult from '$lib/components/propertyPicker/PropPickerResult.svelte'
import { createEventDispatcher, setContext } from 'svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { writable, type Writable } from 'svelte/store'
import type { PickableProperties } from '../previousResults'
export let pickableProperties: Object = {}
export let pickableProperties: PickableProperties | undefined
export let result: any = undefined
export let error: boolean = false
export let displayContext = true
export let priorId: string | undefined
export let notSelectable = false
const propPickerConfig = writable<PropPickerConfig | undefined>(undefined)
const dispatch = createEventDispatcher()
@@ -49,16 +53,21 @@
<slot />
</Pane>
<Pane minSize={20} size={34} class="px-2 py-2 h-full !transition-none">
<PropPicker
{priorId}
{displayContext}
{pickableProperties}
on:select={({ detail }) => {
dispatch('select', detail)
if ($propPickerConfig?.onSelect(detail)) {
propPickerConfig.set(undefined)
}
}}
/>
{#if result}
<PropPickerResult {result} />
{:else if pickableProperties}
<PropPicker
{displayContext}
{error}
{pickableProperties}
{notSelectable}
on:select={({ detail }) => {
dispatch('select', detail)
if ($propPickerConfig?.onSelect(detail)) {
propPickerConfig.set(undefined)
}
}}
/>
{/if}
</Pane>
</Splitpanes>
+3 -5
View File
@@ -33,7 +33,7 @@ export function cleanInputs(flow: Flow | any): Flow {
(x) =>
x != '' &&
!x.startsWith(
`import { previous_result, flow_input, step, variable, resource, params } from 'windmill@`
`import { results, flow_input, variable, resource, params } from 'windmill@`
)
)
.join('\n')
@@ -71,8 +71,6 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{
await inferArgs(mod.language!, mod.content ?? '', schema)
} else if (mod.path && mod.path != '') {
schema = await loadSchema(mod.path!)
console.log(mod.path)
console.log(schema)
} else {
return {
input_transforms: {},
@@ -125,8 +123,8 @@ export function getDefaultExpr(
key: string = 'myfield',
previousExpr?: string
) {
const expr = previousExpr ?? `previous_result.${key}`
return `import { previous_result, flow_input, step, variable, resource, params } from 'windmill${importPath ? `@${importPath}` : ''
const expr = previousExpr ?? `results.${key}`
return `import { results, flow_input, variable, resource, params } from 'windmill${importPath ? `@${importPath}` : ''
}'
${expr}`
@@ -86,11 +86,7 @@
</ul>
{#if level == 0 && topBrackets}<span class="h-0">{closeBracket}</span>{/if}
</span>
<span
class="cursor-pointer hover:bg-gray-200 {level == 0 ? 'ml-2' : ''}"
class:hidden={!collapsed}
on:click={collapse}
>
<span class="cursor-pointer hover:bg-gray-200" class:hidden={!collapsed} on:click={collapse}>
{openBracket}{collapsedSymbol}{closeBracket}
</span>
{#if !isLast && collapsed}
@@ -1,21 +1,21 @@
<script lang="ts">
import { ResourceService, VariableService } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { faClose } from '@fortawesome/free-solid-svg-icons'
import { getContext } from 'svelte'
import Icon from 'svelte-awesome'
import { Button } from '../common'
import { Badge, Button } from '../common'
import type { PropPickerWrapperContext } from '../flows/propPicker/PropPickerWrapper.svelte'
import { createEventDispatcher } from 'svelte'
import ObjectViewer from './ObjectViewer.svelte'
import { keepByKey } from './utils'
import { flowStateStore } from '../flows/flowState'
import { flowIds } from '../flows/flowStore'
import type { PickableProperties } from '../flows/previousResults'
export let pickableProperties: Object = {}
export let pickableProperties: PickableProperties
export let displayContext = true
export let priorId: string | undefined
export let notSelectable: boolean
export let error: boolean = false
$: previousId = pickableProperties?.previousId
let variables: Record<string, string> = {}
let resources: Record<string, any> = {}
let displayVariable = false
@@ -28,23 +28,16 @@
const { propPickerConfig, clearFocus } = getContext<PropPickerWrapperContext>('PropPickerWrapper')
$: propsFiltered =
search === EMPTY_STRING ? pickableProperties : keepByKey(pickableProperties, search)
$: flowInputsFiltered =
search === EMPTY_STRING
? pickableProperties.flow_input
: keepByKey(pickableProperties.flow_input, search)
$: resultByIdFiltered =
search === EMPTY_STRING
? pickableProperties.priorIds
: keepByKey(pickableProperties.priorIds, search)
let priorIds = {}
$: {
if (priorId) {
const allState = $flowStateStore
priorIds = Object.fromEntries(
Object.entries(allState)
.filter(
(o) =>
$flowIds.includes(o[0]) && $flowIds.indexOf(o[0]) <= $flowIds.indexOf(priorId ?? '')
)
.map((o) => [o[0], o[1].previewResult])
)
}
}
async function loadVariables() {
variables = Object.fromEntries(
(
@@ -66,103 +59,131 @@
}
</script>
{#if !notSelectable}
<div class="flex flex-row space-x-1">
{#if $propPickerConfig}
<Badge color="blue">
{`Selected: ${$propPickerConfig?.propName}`}
</Badge>
<Badge color="blue">
{`Mode: ${$propPickerConfig?.insertionMode}`}
</Badge>
{:else}
<Badge>&leftarrow; Select a step input</Badge>
{/if}
</div>
{/if}
<input
type="text"
bind:value={search}
class="bg-gray-50 mt-1 border border-gray-300 text-gray-900 text-sm rounded-lg block px-2 mb-2 w-full"
placeholder="Search prop..."
/>
<div class="flex justify-between items-center space-x-1">
<span class="font-bold text-sm">Step Context</span>
<div class="flex space-x-2 items-center">
{#if $propPickerConfig}
<span
class="flex items-center bg-blue-100 text-blue-800 text-xs font-semibold px-2 py-1 rounded dark:bg-green-200 dark:text-green-900"
>
{`Selected: ${$propPickerConfig?.propName}`}
</span>
<span
class="flex items-center bg-blue-100 text-blue-800 text-xs font-semibold px-2 py-1 rounded dark:bg-green-200 dark:text-green-900"
>
{`Mode: ${$propPickerConfig?.insertionMode}`}
</span>
<button
class="border px-2 py-1 text-xs rounded-md flex items-center hover:bg-gray-50 hover:text-gray-900"
on:click={() => clearFocus()}
>
<Icon data={faClose} class="mr-2" scale={0.8} />
Deselect
</button>
{/if}
<div class:bg-gray-100={!$propPickerConfig && !notSelectable}>
<div class="flex justify-between items-center space-x-1">
<span class="font-bold text-sm">Flow Input</span>
<div class="flex space-x-2 items-center" />
</div>
</div>
<div class="overflow-y-auto mb-2">
<ObjectViewer json={propsFiltered} on:select />
</div>
{#if priorId}
<span class="font-bold text-sm">Result by id</span>
<div class="overflow-y-auto mb-2">
<ObjectViewer
collapsed={true}
json={priorIds}
pureViewer={!$propPickerConfig}
json={flowInputsFiltered}
on:select={(e) => {
const [first, ...second] = e.detail.split('.')
dispatch('select', `result_by_id('${first}')${second.length ? '.' + second.join('.') : ''}`)
dispatch('select', `flow_input.${e.detail}`)
}}
/>
</div>
{/if}
{#if displayContext}
<span class="font-bold text-sm">Variables </span>
<div class="overflow-y-auto mb-2">
{#if displayVariable}
<Button
color="light"
size="xs"
on:click={() => {
displayVariable = false
}}>(-)</Button
>
{#if error}
<span class="font-bold text-sm">Error</span>
<div class="overflow-y-auto mb-2">
<ObjectViewer
rawKey={true}
json={variables}
on:select={(e) => dispatch('select', `variable('${e.detail}')`)}
pureViewer={!$propPickerConfig}
json={{ previous_result: { error: 'The error to handle' } }}
on:select
/>
{:else}
<Button
color="light"
size="xs"
on:click={async () => {
await loadVariables()
displayVariable = true
}}>{'{...}'}</Button
>
</div>
{:else}
{#if previousId}
<span class="font-bold text-sm">Previous Result</span>
<div class="overflow-y-auto mb-2">
<ObjectViewer
pureViewer={!$propPickerConfig}
json={Object.fromEntries(
Object.entries(resultByIdFiltered).filter(([k, v]) => k == previousId)
)}
on:select={(e) => {
dispatch('select', `results.${e.detail}`)
}}
/>
</div>
{/if}
</div>
<span class="font-bold text-sm">Resources</span>
<div class="overflow-y-auto mb-2">
{#if displayResources}
<Button
color="light"
size="xs"
on:click={() => {
displayResources = false
}}>(-)</Button
>
<ObjectViewer
rawKey={true}
json={resources}
on:select={(e) => dispatch('select', `resource('${e.detail}')`)}
/>
{:else}
<Button
color="light"
size="xs"
on:click={async () => {
await loadResources()
displayResources = true
}}>{'{...}'}</Button
>
{#if Object.keys(pickableProperties.priorIds).length > 0}
<span class="font-bold text-sm">All Results</span>
<div class="overflow-y-auto mb-2">
<ObjectViewer
pureViewer={!$propPickerConfig}
collapsed={true}
json={resultByIdFiltered}
on:select={(e) => {
dispatch('select', `results.${e.detail}`)
}}
/>
</div>
{/if}
</div>
{/if}
{/if}
{#if displayContext}
<span class="font-bold text-sm">Variables </span>
<div class="overflow-y-auto mb-2">
{#if displayVariable}
<Button
color="light"
size="xs"
on:click={() => {
displayVariable = false
}}>(-)</Button
>
<ObjectViewer
pureViewer={!$propPickerConfig}
rawKey={true}
json={variables}
on:select={(e) => dispatch('select', `variable('${e.detail}')`)}
/>
{:else}
<button
class="key font-normal rounded px-1 hover:bg-blue-100 !p-0"
on:click={async () => {
await loadVariables()
displayVariable = true
}}>{'{...}'}</button
>
{/if}
</div>
<span class="font-bold text-sm">Resources</span>
<div class="overflow-y-auto mb-2">
{#if displayResources}
<Button
color="light"
size="xs"
on:click={() => {
displayResources = false
}}>(-)</Button
>
<ObjectViewer
pureViewer={!$propPickerConfig}
rawKey={true}
json={resources}
on:select={(e) => dispatch('select', `resource('${e.detail}')`)}
/>
{:else}
<button
class="key font-normal rounded px-1 hover:bg-blue-100 !p-0"
on:click={async () => {
await loadResources()
displayResources = true
}}>{'{...}'}</button
>
{/if}
</div>
{/if}
</div>
@@ -0,0 +1,16 @@
<script lang="ts">
import { createEventDispatcher } from 'svelte'
import ObjectViewer from './ObjectViewer.svelte'
export let result: any
const dispatch = createEventDispatcher()
</script>
<div class="w-full">
<span class="font-bold text-sm">Result</span>
<div class="overflow-y-auto mb-2 w-full">
<ObjectViewer json={{ result }} on:select />
</div>
</div>
@@ -19,7 +19,7 @@ function diff(target: Object, source: Object): Object {
const result = {}
Object.keys(target).forEach((key: string) => {
Object.keys(target ?? {}).forEach((key: string) => {
if (typeof source[key] === 'object') {
const difference = diff(target[key], source[key])
@@ -34,7 +34,10 @@ function diff(target: Object, source: Object): Object {
return result
}
export function keepByKey(json: Object, key: string): Object {
export function keepByKey(json: Object | undefined, key: string): Object {
if (!json) {
return {}
}
return diff(json, filterByKey(json, key))
}
-61
View File
@@ -325,44 +325,6 @@ export function mapUserToUserExt(user: User): UserExt {
}
}
export function buildExtraLib(flowInput: string, previousResultType?: string): string {
return `
/**
* get variable (including secret) at path
* @param {string} path - path of the variable (e.g: g/all/pretty_secret)
*/
export function variable(path: string): string;
/**
* get resource at path
* @param {string} path - path of the resource (e.g: g/all/my_resource)
*/
export function resource(path: string): any;
/**
* get result of step n.
* If n is negative, for instance -1, it is the step just before this one.
* Step 0 is flow input.
* @param {number} n - step number.
*/
export function step(n: number): any;
/**
* flow input as an object
*/
export const flow_input: ${flowInput};
/**
* previous result as an object
*/
export const previous_result: ${previousResultType || 'any'};
/**
* static params of this same step
*/
export const params: any;`
}
export function schemaToTsType(schema: Schema): string {
if (!schema || !schema.properties) {
return 'any'
@@ -411,29 +373,6 @@ export function schemaToObject(schema: Schema, args: Record<string, any>): Objec
return object
}
export function valueToTsType(value: any): string {
const typeOfValue: string = typeof value
if (['string', 'number', 'boolean'].includes(typeOfValue)) {
return typeOfValue
} else if (Array.isArray(value)) {
const type = objectToTsType(value[0])
return `Array<${type}>`
} else if (typeof value === 'object') {
return objectToTsType(value)
} else {
return 'any'
}
}
export function objectToTsType(object: Object): string {
if (!object) {
return 'any'
}
const propKeys = Object.keys(object)
const types = propKeys.map((key: string) => `${key}: ${valueToTsType(object[key])}`).join(';')
return `{ ${types} }`
}
export type InputCat =
| 'string'
@@ -222,7 +222,7 @@
<div class="mt-4">
<FlowViewer {flow} noSummary={true} />
<h2 id="webhook" class="mb-4 mt-10 text-gray-700 pb-1 mb-3 border-b"
<h2 id="webhook" class="mt-10 text-gray-700 pb-1 mb-3 border-b"
>Webhook<Tooltip
>To trigger this script with a webhook, do a POST request to the endpoint below. Flows
are not public and can only be run by users with at least view rights on them. You