starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-12 00:06:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

restructure the entire backend layout using workspaces (#815)

Browse Source
This commit is contained in:
Kai Jellinghaus
2022-10-29 11:58:06 +02:00
committed by GitHub
parent 687fc102d5
commit 6263e90735
89 changed files with 9008 additions and 8974 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/DockerfileBackendTests
+4 -3
View File
@@ -30,7 +30,6 @@ RUN apt-get -y update \
ENV SQLX_OFFLINE=true
COPY ./nsjail /nsjail
RUN mkdir -p /frontend/build
RUN apt-get update \
@@ -38,7 +37,7 @@ RUN apt-get update \
make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
libv8-dev tesseract-ocr golang-go \
libv8-dev tesseract-ocr nodejs npm\
&& rm -rf /var/lib/apt/lists/*
RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
@@ -62,4 +61,6 @@ COPY --from=nsjail /nsjail/nsjail /bin/nsjail
COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
RUN apt-get update \
&& apt-get install -y postgresql-client
&& apt-get install -y postgresql-client --allow-unauthenticated
RUN rustup component add rustfmt
.github/change-versions.sh
+1 -1
View File
@@ -4,7 +4,7 @@ VERSION=$1
echo "Updating versions to: $VERSION"
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
sed -i -e "/version: /s/: .*/: $VERSION/" backend/openapi.yaml
sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill/pyproject.toml
.github/workflows/backend-test.yml
+4 -2
View File
@@ -35,7 +35,9 @@ jobs:
- uses: actions/checkout@v3
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend -> target
workspaces: |
backend
backend -> target
- name: cargo test
timeout-minutes: 5
run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test -- --nocapture
run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
Dockerfile
+52 -13
View File
@@ -29,7 +29,7 @@ RUN npm ci
# Copy all local files into the image.
COPY frontend .
RUN mkdir /backend
COPY /backend/openapi.yaml /backend/openapi.yaml
COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
COPY /openflow.openapi.yaml /openflow.openapi.yaml
RUN npm run generate-backend-client
ENV NODE_OPTIONS "--max-old-space-size=8192"
@@ -38,29 +38,68 @@ RUN npm run check
FROM rust:slim-buster as builder
RUN apt-get update && apt-get install -y git libssl-dev pkg-config
RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
RUN apt-get -y update \
&& apt-get install -y \
curl lld nodejs npm
RUN rustup component add rustfmt
RUN USER=root cargo new --bin windmill
WORKDIR /windmill
COPY ./backend/Cargo.toml .
COPY ./backend/Cargo.lock .
COPY ./backend/.cargo/ .cargo/
COPY ./openflow.openapi.yaml /openflow.openapi.yaml
RUN apt-get -y update \
&& apt-get install -y \
curl lld
RUN USER=root cargo new --bin windmill
RUN USER=root cargo new --lib windmill-api
RUN USER=root cargo new --lib windmill-audit
RUN USER=root cargo new --lib windmill-queue
RUN USER=root cargo new --lib windmill-worker
WORKDIR /windmill/parsers
RUN USER=root cargo new --lib windmill-parser
RUN USER=root cargo new --lib windmill-parser-go
RUN USER=root cargo new --lib windmill-parser-py
RUN USER=root cargo new --lib windmill-parser-ts
WORKDIR /windmill
ENV SQLX_OFFLINE=true
# COPY ./backend/Cargo.toml .
# COPY ./backend/windmill-api/Cargo.toml ./windmill-api/
# COPY ./backend/windmill-audit/Cargo.toml ./windmill-audit/
# COPY ./backend/sqlx-data.json ./
# COPY ./backend/windmill-common ./windmill-common
# COPY ./backend/windmill-queue/Cargo.toml ./windmill-common/
# COPY ./backend/windmill-queue/Cargo.toml ./windmill-queue/
# COPY ./backend/windmill-worker/Cargo.toml ./windmill-worker/
# COPY ./backend/parsers/windmill-parser/Cargo.toml ./parsers/windmill-parser/
# COPY ./backend/parsers/windmill-parser-go/Cargo.toml ./parsers/windmill-parser-go/
# COPY ./backend/parsers/windmill-parser-py/Cargo.toml ./parsers/windmill-parser-py/
# COPY ./backend/parsers/windmill-parser-ts/Cargo.toml ./parsers/windmill-parser-ts/
# COPY ./backend/.cargo/ .cargo/
# COPY ./backend/windmill-api-client/ ./windmill-api-client/
# COPY ./backend/windmill-api/openapi.yaml ./windmill-api/openapi.yaml
ENV CARGO_INCREMENTAL=1
RUN cargo build --release
RUN rm src/*.rs
# RUN cargo build --release
# RUN rm ./src/*.rs
# RUN rm ./windmill-api/src/*.rs
# RUN rm ./windmill-api-client/src/*.rs
# RUN rm ./windmill-audit/src/*.rs
# RUN rm ./windmill-common/src/*.rs
# RUN rm ./windmill-queue/src/*.rs
# RUN rm ./windmill-worker/src/*.rs
# RUN rm ./parsers/windmill-parser/src/*.rs
# RUN rm ./parsers/windmill-parser-go/src/*.rs
# RUN rm ./parsers/windmill-parser-py/src/*.rs
# RUN rm ./parsers/windmill-parser-ts/src/*.rs
RUN rm ./target/release/deps/windmill*
ENV SQLX_OFFLINE=true
# RUN rm -r ./target/release/deps/windmill*
COPY ./backend ./
COPY ./nsjail /nsjail
COPY --from=frontend /frontend /frontend
COPY .git/ .git/
backend/Cargo.lock
Generated
+804 -603
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+80 -14
View File
@@ -1,13 +1,65 @@
[package]
name = "windmill"
version = "1.41.0"
authors.workspace = true
edition.workspace = true
[workspace]
members = [
"./windmill-api",
"./windmill-queue",
"./windmill-worker",
"./windmill-common",
"./windmill-audit",
"./windmill-api-client",
"./parsers/windmill-parser",
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-go",
"./parsers/windmill-parser-py",
]
[workspace.package]
version = "1.41.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
[build-dependencies]
deno_core = "^0"
[[bin]]
name = "windmill"
path = "./src/main.rs"
[dependencies]
anyhow.workspace = true
tokio.workspace = true
dotenv.workspace = true
windmill-common = { workspace = true, features = ["tracing_init"] }
windmill-api.workspace = true
windmill-api-client.workspace = true
windmill-worker.workspace = true
futures.workspace = true
tracing.workspace = true
sqlx.workspace = true
tokio-metrics.workspace = true
rand.workspace = true
chrono.workspace = true
[dev-dependencies]
serde_json.workspace = true
reqwest.workspace = true
windmill-queue.workspace = true
axum.workspace = true
[workspace.dependencies]
windmill-api = { path = "./windmill-api" }
windmill-api-client = { path = "./windmill-api-client" }
windmill-queue = { path = "./windmill-queue" }
windmill-worker = { path = "./windmill-worker" }
windmill-common = { path = "./windmill-common" }
windmill-audit = { path = "./windmill-audit" }
windmill-parser = { path = "./parsers/windmill-parser" }
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
windmill-parser-go = { path = "./parsers/windmill-parser-go" }
tiny_http = "0.12.0"
axum = { version = "^0", features = ["headers"] }
headers = "^0"
hyper = { version = "^0", features = ["full"] }
@@ -20,31 +72,36 @@ serde_json = { version = "^1", features = ["preserve_order"] }
uuid = { version = "^1", features = ["serde", "v4"] }
thiserror = "^1"
anyhow = "^1"
chrono = { version = "^0", features = ["serde"]}
chrono = { version = "^0", features = ["serde"] }
tracing = "^0"
tracing-subscriber = { version = "^0", features = ["env-filter", "json"]}
tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
console-subscriber = "^0"
prometheus = { version = "^0", default-features = false }
phf = { version = "0.11", features = ["macros"] }
rust-embed = "^6"
mime_guess = "^2"
hex = "^0"
sql-builder = "^3"
argon2 = "^0"
retainer = "^0"
rand = "^0"
rand = "0.8.5"
rand_core = { version = "^0", features = ["std"] }
magic-crypt = "^3"
git-version = "^0"
rustpython-parser = "^0"
rustpython-parser = { git = "https://github.com/RustPython/RustPython" }
cron = "^0"
lettre = { version = "^0", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"], default-features = false}
lettre = { version = "^0", features = [
"rustls-tls",
"tokio1",
"tokio1-rustls-tls",
"builder",
"smtp-transport",
], default-features = false }
urlencoding = "^2"
url = "^2"
async-oauth2 = "^0"
reqwest = { version = "^0", features = ["json"] }
time = "^0"
time = "0.3.16"
serde_urlencoded = "^0"
tokio-tar = "^0"
tempfile = "^3"
@@ -57,14 +114,23 @@ async-recursion = "^1"
swc_common = "^0"
swc_ecma_parser = "^0"
swc_ecma_ast = "^0"
base64 = "^0"
base64 = "^0"
unicode-general-category = "^0"
hmac = "^0"
sha2 = "^0"
sqlx = { version = "^0", features = ["offline", "macros", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
hmac = "0.12.1"
sha2 = "0.10.6"
sqlx = { version = "^0", features = [
"offline",
"macros",
"migrate",
"uuid",
"json",
"chrono",
"postgres",
"runtime-tokio-rustls",
] }
dotenv = "^0"
ulid = { version = "^1", features = ["uuid"] }
futures = "^0"
tokio-metrics = "0.1.0"
lazy_static = "1.4.0"
serde_derive = "1.0.147"
backend/README.md
+15
View File
@@ -0,0 +1,15 @@
# Windmill Backend
This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary.
## Components
| name | description |
| --------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
| [windmill-api](./windmill-api/) | The API server, exposing functionality to other components and the frontend |
| [windmill-api-client](./windmill-api-client/) | An autogenerated Rust API client, used by other components to talk to the API |
| [windmill-audit](./windmill-audit/) | Contains audit functionality, allowing different components to record important actions |
| [windmill-common](./windmill-common/) | Common code shared by all crates |
| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. |
| [parsers](./parsers/) | Contains code to parse signatures in different langauges. |
backend/parsers/windmill-parser-go/Cargo.toml
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "windmill-parser-go"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_go"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-common.workspace = true
phf.workspace = true
unicode-general-category.workspace = true
itertools.workspace = true
anyhow.workspace = true
backend/src/parser_go.rs → backend/parsers/windmill-parser-go/src/lib.rs
+12 -9
View File
@@ -1,16 +1,19 @@
#![allow(non_snake_case)] // TODO: switch to parse_* function naming
mod parser_go_ast;
mod parser_go_scanner;
mod parser_go_token;
use itertools::Itertools;
use crate::error::to_anyhow;
use crate::parser::{Arg, MainArgSignature, ObjectProperty, Typ};
use crate::parser_go_ast::{self, FieldList, Ident, StructType};
use crate::parser_go_ast::{Decl, Expr};
use crate::parser_go_scanner;
use crate::parser_go_token::{Position, Token};
use parser_go_ast::{Decl, Expr};
use parser_go_ast::{FieldList, Ident, StructType};
use parser_go_token::{Position, Token};
use std::fmt;
use windmill_common::error::to_anyhow;
use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
pub fn parse_go_sig(code: &str) -> crate::error::Result<MainArgSignature> {
pub fn parse_go_sig(code: &str) -> windmill_common::error::Result<MainArgSignature> {
let filtered_code = filter_non_main(code);
let file = parse_file("main.go", &filtered_code).map_err(to_anyhow)?;
if let Some(Decl::FuncDecl(func)) = file.decls.first() {
@@ -26,7 +29,7 @@ pub fn parse_go_sig(code: &str) -> crate::error::Result<MainArgSignature> {
.collect_vec();
Ok(MainArgSignature { star_args: false, star_kwargs: false, args })
} else {
Err(crate::error::Error::BadRequest(
Err(windmill_common::error::Error::BadRequest(
"no main function found".to_string(),
))
}
@@ -107,7 +110,7 @@ pub fn otyp_to_string(otyp: Option<String>) -> String {
#[cfg(test)]
mod tests {
use crate::parser::{Arg, MainArgSignature, ObjectProperty, Typ};
use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
use super::*;
backend/src/parser_go_ast.rs → backend/parsers/windmill-parser-go/src/parser_go_ast.rs
View File
backend/src/parser_go_scanner.rs → backend/parsers/windmill-parser-go/src/parser_go_scanner.rs
View File
backend/src/parser_go_token.rs → backend/parsers/windmill-parser-go/src/parser_go_token.rs
View File
backend/parsers/windmill-parser-py/Cargo.toml
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "windmill-parser-py"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_py"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-common.workspace = true
rustpython-parser.workspace = true
phf.workspace = true
itertools.workspace = true
regex.workspace = true
serde_json.workspace = true
anyhow.workspace = true
backend/src/parser_py.rs → backend/parsers/windmill-parser-py/src/lib.rs
+90 -75
View File
@@ -11,21 +11,19 @@ use std::collections::HashMap;
use itertools::Itertools;
use phf::phf_map;
use regex::Regex;
use serde_json::json;
use crate::{
error,
parser::{Arg, MainArgSignature, Typ},
};
use serde_json::json;
use windmill_common::error;
use windmill_parser::{Arg, MainArgSignature, Typ};
use rustpython_parser::{
ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
ast::{Constant, ExprKind, Located, StmtKind},
parser,
};
fn filter_non_main(code: &str) -> String {
const DEF_MAIN: &str = "def main(";
const DEF_MAIN: &str = "def main(";
fn filter_non_main(code: &str) -> String {
let mut filtered_code = String::new();
let mut code_iter = code.split("\n");
let mut remaining: String = String::new();
@@ -66,35 +64,27 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
"No main function found".to_string(),
));
}
let ast = parser::parse_program(&filtered_code)
.map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
.statements;
let ast = parser::parse_program(&filtered_code, "main.py").map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
})?;
let param = ast.into_iter().find_map(|x| match x {
Located {
location: _,
node:
StatementType::FunctionDef {
is_async: _,
name,
args,
body: _,
decorator_list: _,
returns: _,
},
} if &name == "main" => Some(*args),
Located { node: StmtKind::FunctionDef { name, args, .. }, .. } if &name == "main" => {
Some(*args)
}
_ => None,
});
if let Some(params) = param {
//println!("{:?}", params);
let def_arg_start = params.args.len() - params.defaults.len();
Ok(MainArgSignature {
star_args: params.vararg != Varargs::None,
star_kwargs: params.vararg != Varargs::None,
star_args: params.vararg.is_some(),
star_kwargs: params.vararg.is_some(),
args: params
.args
.into_iter()
.enumerate()
.map(|(i, x)| {
let x = x.node;
let default = if i >= def_arg_start {
to_value(&params.defaults[i - def_arg_start].node)
} else {
@@ -104,20 +94,18 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
otyp: None,
name: x.arg,
typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
Located { location: _, node: ExpressionType::Identifier { name } } => {
match name.as_ref() {
"str" => Typ::Str(None),
"float" => Typ::Float,
"int" => Typ::Int,
"bool" => Typ::Bool,
"dict" => Typ::Object(vec![]),
"list" => Typ::List(Box::new(Typ::Str(None))),
"bytes" => Typ::Bytes,
"datetime" => Typ::Datetime,
"datetime.datetime" => Typ::Datetime,
_ => Typ::Unknown,
}
}
Located { node: ExprKind::Name { id, .. }, .. } => match id.as_ref() {
"str" => Typ::Str(None),
"float" => Typ::Float,
"int" => Typ::Int,
"bool" => Typ::Bool,
"dict" => Typ::Object(vec![]),
"list" => Typ::List(Box::new(Typ::Str(None))),
"bytes" => Typ::Bytes,
"datetime" => Typ::Datetime,
"datetime.datetime" => Typ::Datetime,
_ => Typ::Unknown,
},
_ => Typ::Unknown,
}),
has_default: default.is_some(),
@@ -133,24 +121,15 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
}
}
fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
fn to_value(et: &ExprKind) -> Option<serde_json::Value> {
match et {
ExpressionType::String { value: StringGroup::Constant { value } } => Some(json!(value)),
ExpressionType::Number { value } => match value {
Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
Number::Float { value } => Some(json!(value)),
_ => None,
},
ExpressionType::True => Some(json!(true)),
ExpressionType::False => Some(json!(false)),
ExpressionType::Dict { elements } => {
let v = elements
ExprKind::Constant { value, .. } => Some(constant_to_value(value)),
ExprKind::Dict { keys, values } => {
let v = keys
.into_iter()
.zip(values)
.map(|(k, v)| {
let key = k
.as_ref()
.and_then(|x| to_value(&x.node))
let key = to_value(&k.node)
.and_then(|x| match x {
serde_json::Value::String(s) => Some(s),
_ => None,
@@ -161,23 +140,32 @@ fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
.collect::<HashMap<String, _>>();
Some(json!(v))
}
ExpressionType::List { elements } => {
let v = elements
ExprKind::List { elts, .. } => {
let v = elts
.into_iter()
.map(|x| to_value(&x.node))
.collect::<Vec<_>>();
Some(json!(v))
}
ExpressionType::None => Some(json!(null)),
ExpressionType::Call { function: _, args: _, keywords: _ } => {
Some(json!("<function call>"))
}
ExprKind::Call { .. } => Some(json!("<function call>")),
_ => None,
}
}
fn constant_to_value(c: &Constant) -> serde_json::Value {
match c {
Constant::None => json!(null),
Constant::Bool(b) => json!(b),
Constant::Str(s) => json!(s),
Constant::Bytes(b) => json!(b),
Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")),
Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::<Vec<_>>()),
Constant::Float(f) => json!(f),
Constant::Complex { real, imag } => json!([real, imag]),
Constant::Ellipsis => json!("..."),
}
}
static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! {
"psycopg2" => "psycopg2-binary"
};
@@ -206,25 +194,22 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
.collect();
Ok(lines)
} else {
let code = &&code;
let ast = parser::parse_program(code)
.map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
})?
.statements;
let code = code.split(DEF_MAIN).next().unwrap_or("");
let ast = parser::parse_program(code, "main.py").map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
})?;
let imports = ast
.into_iter()
.filter_map(|x| match x {
Located { location: _, node } => match node {
StatementType::Import { names } => Some(
Located { node, .. } => match node {
StmtKind::Import { names } => Some(
names
.into_iter()
.map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
.map(|x| x.node.name.split('.').next().unwrap_or("").to_string())
.map(replace_import)
.collect::<Vec<String>>(),
),
StatementType::ImportFrom { level: _, module: Some(mod_), names: _ } => {
StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => {
let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
Some(vec![replace_import(imprt)])
@@ -244,6 +229,8 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
#[test]
@@ -252,7 +239,7 @@ mod tests {
import os
def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1), f = \"wewe\", g = 21, h = [1,2], i = True):
print(f\"Hello World and a warm welcome especially to {name}\")
print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
@@ -286,7 +273,35 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
typ: Typ::Bytes,
default: Some(json!("<function call>")),
has_default: true
}
},
Arg {
otyp: None,
name: "f".to_string(),
typ: Typ::Unknown,
default: Some(json!("wewe")),
has_default: true
},
Arg {
otyp: None,
name: "g".to_string(),
typ: Typ::Unknown,
default: Some(json!(21)),
has_default: true
},
Arg {
otyp: None,
name: "h".to_string(),
typ: Typ::Unknown,
default: Some(json!([1, 2])),
has_default: true
},
Arg {
otyp: None,
name: "i".to_string(),
typ: Typ::Unknown,
default: Some(json!(true)),
has_default: true
},
]
}
);
backend/parsers/windmill-parser-ts/Cargo.toml
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "windmill-parser-ts"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_ts"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-common.workspace = true
deno_core.workspace = true
swc_common.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
serde_json.workspace = true
anyhow.workspace = true
backend/src/parser_ts.rs → backend/parsers/windmill-parser-ts/src/lib.rs
+22 -6
View File
@@ -5,12 +5,9 @@
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
error,
js_eval::eval_sync,
parser::{Arg, MainArgSignature, ObjectProperty, Typ},
};
use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
use windmill_common::error;
use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
use serde_json::Value;
use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
@@ -263,6 +260,25 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
}
}
pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
let mut context = JsRuntime::new(RuntimeOptions::default());
let code = format!("let x = {}; x", code);
let res = context.execute_script("<anon>", &code);
match res {
Ok(global) => {
let scope = &mut context.handle_scope();
let local = v8::Local::new(scope, global);
let deserialized_value = serde_v8::from_v8::<serde_json::Value>(scope, local);
match deserialized_value {
Ok(value) => Ok(value),
Err(err) => Err(format!("Cannot deserialize value: {:?}", err)),
}
}
Err(err) => Err(format!("Evaling error: {:?}", err)),
}
}
#[cfg(test)]
mod tests {
backend/parsers/windmill-parser/Cargo.toml
+13
View File
@@ -0,0 +1,13 @@
[package]
name = "windmill-parser"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser"
path = "./src/lib.rs"
[dependencies]
serde.workspace = true
serde_json.workspace = true
backend/src/parser.rs → backend/parsers/windmill-parser/src/lib.rs
View File
backend/src/client.rs
-47
View File
@@ -1,47 +0,0 @@
use crate::{error::Error, variables::ListableVariable};
pub async fn get_variable(
workspace: &str,
path: &str,
token: &str,
base_url: &str,
) -> Result<String, anyhow::Error> {
let client = reqwest::Client::new();
let res = client
.get(format!("{base_url}/api/w/{workspace}/variables/get/{path}"))
.bearer_auth(token)
.send()
.await?;
if res.status().is_success() {
let value = res
.json::<ListableVariable>()
.await?
.value
.unwrap_or_else(|| "".to_string());
Ok(value)
} else {
Err(Error::NotFound(format!("Variable not found at {path}")))?
}
}
pub async fn get_resource(
workspace: &str,
path: &str,
token: &str,
base_url: &str,
) -> Result<Option<serde_json::Value>, anyhow::Error> {
let client = reqwest::Client::new();
let res = client
.get(format!(
"{base_url}/api/w/{workspace}/resources/get_value/{path}"
))
.bearer_auth(token)
.send()
.await?;
if res.status().is_success() {
let value = res.json::<Option<serde_json::Value>>().await?;
Ok(value)
} else {
Err(Error::NotFound(format!("Resource not found at {path}")))?
}
}
backend/src/main.rs
+89 -20
View File
@@ -8,21 +8,22 @@
use std::net::SocketAddr;
use dotenv::dotenv;
use windmill::WorkerConfig;
use sqlx::{Pool, Postgres};
use windmill_common::utils::rd_string;
use windmill_worker::WorkerConfig;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
dotenv().ok();
dotenv::dotenv().ok();
windmill::initialize_tracing();
windmill_common::tracing_init::initialize_tracing();
let db = windmill::connect_db().await?;
let db = windmill_common::connect_db().await?;
let num_workers = std::env::var("NUM_WORKERS")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill::DEFAULT_NUM_WORKERS as i32);
.unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32);
let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
.ok()
@@ -40,39 +41,40 @@ async fn main() -> anyhow::Result<()> {
.unwrap_or(false);
if server_mode {
windmill::migrate_db(&db).await?;
windmill_api::migrate_db(&db).await?;
}
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
let shutdown_signal = windmill::shutdown_signal(tx);
let shutdown_signal = windmill_common::shutdown_signal(tx);
let base_internal_url =
std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
let timeout = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill_common::DEFAULT_TIMEOUT);
if server_mode || num_workers > 0 {
let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
let timeout = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill::DEFAULT_TIMEOUT);
let base_url_2 = base_url.clone();
let base_url2 = base_url.clone();
let server_f = async {
if server_mode {
windmill::run_server(db.clone(), addr, base_url_2, rx.resubscribe()).await?;
windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?;
}
Ok(()) as anyhow::Result<()>
};
let base_url = base_url2.clone();
let workers_f = async {
if num_workers > 0 {
let sleep_queue = std::env::var("SLEEP_QUEUE")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(windmill::DEFAULT_SLEEP_QUEUE);
.unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE);
let disable_nuser = std::env::var("DISABLE_NUSER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
@@ -91,7 +93,7 @@ async fn main() -> anyhow::Result<()> {
{base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
{timeout}, KEEP_JOB_DIR: {keep_job_dir}"
);
windmill::run_workers(
run_workers(
db.clone(),
addr,
timeout,
@@ -111,24 +113,91 @@ async fn main() -> anyhow::Result<()> {
Ok(()) as anyhow::Result<()>
};
let base_url = base_url2;
let monitor_f = async {
if server_mode {
windmill::monitor_db(&db, timeout, rx.resubscribe());
monitor_db(&db, timeout, base_url, rx.resubscribe());
}
Ok(()) as anyhow::Result<()>
};
let metrics_f = async {
match metrics_addr {
Some(addr) => windmill::serve_metrics(addr, rx.resubscribe())
Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
.await
.map_err(anyhow::Error::from),
None => Ok(()),
}
};
futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f, metrics_f)?;
futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
}
Ok(())
}
pub fn monitor_db(
db: &Pool<Postgres>,
timeout: i32,
base_url: String,
rx: tokio::sync::broadcast::Receiver<()>,
) {
let db1 = db.clone();
let db2 = db.clone();
let rx2 = rx.resubscribe();
tokio::spawn(async move {
windmill_worker::handle_zombie_jobs_periodically(&db1, timeout, &base_url, rx).await
});
tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
}
pub async fn run_workers(
db: Pool<Postgres>,
addr: SocketAddr,
timeout: i32,
num_workers: i32,
sleep_queue: u64,
worker_config: WorkerConfig,
rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<()> {
let instance_name = rd_string(5);
let monitor = tokio_metrics::TaskMonitor::new();
let ip = windmill_common::external_ip::get_ip()
.await
.unwrap_or_else(|e| {
tracing::warn!(error = e.to_string(), "failed to get external IP");
"unretrievable IP".to_string()
});
let mut handles = Vec::with_capacity(num_workers as usize);
for i in 1..(num_workers + 1) {
let db1 = db.clone();
let instance_name = instance_name.clone();
let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
let ip = ip.clone();
let rx = rx.resubscribe();
let worker_config = worker_config.clone();
handles.push(tokio::spawn(monitor.instrument(async move {
tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
windmill_worker::run_worker(
&db1,
timeout,
&instance_name,
worker_name,
i as u64,
num_workers as u64,
&ip,
sleep_queue,
worker_config,
rx,
)
.await
})));
}
futures::future::try_join_all(handles).await?;
Ok(())
}
backend/src/users.rs
-1549
View File
File diff suppressed because it is too large Load Diff
backend/src/worker.rs
-3934
View File
File diff suppressed because it is too large Load Diff
backend/src/fixtures/base.sql → backend/tests/fixtures/base.sql
Vendored
View File
backend/tests/worker.rs
+1905
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api-client/.gitignore
+1
View File
@@ -0,0 +1 @@
bundled.json
backend/windmill-api-client/Cargo.toml
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "windmill-api-client"
version.workspace = true
authors.workspace = true
edition.workspace = true
build = "build.rs"
[lib]
name = "windmill_api_client"
path = "./src/lib.rs"
[dependencies]
progenitor-client = { git = "https://github.com/oxidecomputer/progenitor" }
reqwest = { version = "0.11", features = ["json", "stream"] }
serde = { version = "1.0", features = ["derive"] }
chrono.workspace = true
uuid.workspace = true
serde_json.workspace = true
rand.workspace = true
base64.workspace = true
[build-dependencies]
progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
serde_json = "1.0"
backend/windmill-api-client/README.md
+8
View File
@@ -0,0 +1,8 @@
# Windmill API Client
This holds an autogenerated OpenAPI client in Rust. It's exclusively used in the backend to talk to the [api server](../windmill-api/).
## Generate
Simply run `sh bundle.sh` to update bundled.json. The source code will automatically update.
This requires the swagger-cli to be installed for bundling.
backend/windmill-api-client/build.rs
+22
View File
@@ -0,0 +1,22 @@
use std::{
env,
fs::{self, File},
path::Path,
process::Command,
};
fn main() {
let src = "../windmill-api/openapi.yaml";
println!("cargo:rerun-if-changed={}", src);
Command::new("sh").args(&["bundle.sh"]).status().unwrap();
let file = File::open("./bundled.json").unwrap();
let spec = serde_json::from_reader(file).unwrap();
let mut generator = progenitor::Generator::default();
let content = generator.generate_text(&spec).unwrap();
let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf();
out_file.push("codegen.rs");
fs::write(out_file, content).unwrap();
}
backend/windmill-api-client/bundle.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
npx swagger-cli bundle ../windmill-api/openapi.yaml > bundled.json
backend/windmill-api-client/src/lib.rs
+14
View File
@@ -0,0 +1,14 @@
include!(concat!(env!("OUT_DIR"), "/codegen.rs"));
pub fn create_client(base_url: &str, token: String) -> Client {
let mut val = reqwest::header::HeaderValue::from_str(&format!("Bearer {token}"))
.expect("header creation");
val.set_sensitive(true);
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(reqwest::header::AUTHORIZATION, val);
let client = reqwest::ClientBuilder::new()
.default_headers(headers)
.build()
.expect("client build");
Client::new_with_client(&format!("{}/api", base_url.trim_end_matches('/')), client)
}
backend/windmill-api/Cargo.toml
+64
View File
@@ -0,0 +1,64 @@
[package]
name = "windmill-api"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_api"
path = "src/lib.rs"
[[bin]]
name = "windmill_api"
path = "src/main.rs"
[dependencies]
windmill-queue.workspace = true
windmill-common = { workspace = true, features = [
"reqwest",
"prometheus",
"axum",
"tokio",
"hyper",
"sqlx",
"tracing_init",
] }
windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-go.workspace = true
windmill-parser-py.workspace = true
tokio.workspace = true
anyhow.workspace = true
argon2.workspace = true
axum.workspace = true
futures.workspace = true
git-version.workspace = true
tower.workspace = true
tower-cookies.workspace = true
tower-http.workspace = true
hyper.workspace = true
itertools.workspace = true
reqwest.workspace = true
serde.workspace = true
sqlx.workspace = true
async-oauth2.workspace = true
tracing.workspace = true
sql-builder.workspace = true
serde_json.workspace = true
chrono.workspace = true
hex.workspace = true
base64.workspace = true
serde_urlencoded.workspace = true
cron.workspace = true
mime_guess.workspace = true
rust-embed.workspace = true
tracing-subscriber.workspace = true
retainer.workspace = true
rand.workspace = true
time.workspace = true
magic-crypt.workspace = true
tempfile.workspace = true
tokio-util.workspace = true
tokio-tar.workspace = true
hmac.workspace = true
backend/windmill-api/README.md
+5
View File
@@ -0,0 +1,5 @@
# Windmill API
The API server, exposing functionality to other components and the frontend
This crate exposes both a library as well as a binary target.
backend/openapi.yaml → backend/windmill-api/openapi.yaml
+41 -21
View File
@@ -1608,7 +1608,7 @@ paths:
type: object
properties:
flow:
$ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow"
$ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow"
/scripts/hub/get/{path}:
get:
@@ -1936,7 +1936,6 @@ paths:
schema:
type: string
/w/{workspace}/scripts/exists/p/{path}:
get:
summary: exists script by path
@@ -2086,6 +2085,36 @@ paths:
application/json:
schema: {}
/w/{workspace}/jobs/result_by_id/{flow_job_id}/{node_id}:
get:
summary: get job result by id
operationId: resultById
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: flow_job_id
in: path
required: true
schema:
type: string
- name: node_id
in: path
required: true
schema:
type: string
- name: skip_direct
description: Skip checking that the node is part of the given flow.
in: query
schema:
type: boolean
responses:
"200":
description: job result
content:
application/json:
schema: {}
/w/{workspace}/flows/list:
get:
summary: list all available flows
@@ -2601,7 +2630,6 @@ paths:
schema:
type: string
/w/{workspace}/jobs/resume/{id}/{resume_id}/{signature}:
get:
summary: resume a job for a suspended flow
@@ -2621,10 +2649,6 @@ paths:
required: true
schema:
type: string
- name: payload
in: query
schema:
type: object
- name: approver
in: query
schema:
@@ -2692,10 +2716,6 @@ paths:
required: true
schema:
type: string
- name: payload
in: query
schema:
type: object
- name: approver
in: query
schema:
@@ -3496,7 +3516,7 @@ components:
# explode: false
schemas:
$ref: "../openflow.openapi.yaml#/components/schemas"
$ref: "../../openflow.openapi.yaml#/components/schemas"
Script:
type: object
properties:
@@ -3616,7 +3636,7 @@ components:
"flow",
"flowpreview",
"script_hub",
"identity"
"identity",
]
schedule_path:
type: string
@@ -3626,9 +3646,9 @@ components:
The user (u/userfoo) or group (g/groupfoo) whom
the execution of this script will be permissioned_as and by extension its DT_TOKEN.
flow_status:
$ref: "../openflow.openapi.yaml#/components/schemas/FlowStatus"
$ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus"
raw_flow:
$ref: "../openflow.openapi.yaml#/components/schemas/FlowValue"
$ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue"
is_flow_step:
type: boolean
language:
@@ -3695,7 +3715,7 @@ components:
"flow",
"flowpreview",
"script_hub",
"identity"
"identity",
]
schedule_path:
type: string
@@ -3705,9 +3725,9 @@ components:
The user (u/userfoo) or group (g/groupfoo) whom
the execution of this script will be permissioned_as and by extension its DT_TOKEN.
flow_status:
$ref: "../openflow.openapi.yaml#/components/schemas/FlowStatus"
$ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus"
raw_flow:
$ref: "../openflow.openapi.yaml#/components/schemas/FlowValue"
$ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue"
is_flow_step:
type: boolean
language:
@@ -4379,7 +4399,7 @@ components:
Flow:
allOf:
- $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow"
- $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow"
- $ref: "#/components/schemas/FlowMetadata"
FlowMetadata:
@@ -4409,7 +4429,7 @@ components:
OpenFlowWPath:
allOf:
- $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow"
- $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow"
- type: object
properties:
path:
@@ -4421,7 +4441,7 @@ components:
type: object
properties:
value:
$ref: "../openflow.openapi.yaml#/components/schemas/FlowValue"
$ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue"
path:
type: string
args:
backend/windmill-api/src/audit.rs
+44
View File
@@ -0,0 +1,44 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{
extract::{Path, Query},
routing::get,
Extension, Json, Router,
};
use windmill_audit::{AuditLog, ListAuditLogQuery};
use windmill_common::{error::JsonResult, utils::Pagination};
use crate::{db::UserDB, users::Authed};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_audit))
.route("/get/:id", get(get_audit))
}
async fn get_audit(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(id): Path<i32>,
) -> JsonResult<AuditLog> {
let tx = user_db.begin(&authed).await?;
let audit = windmill_audit::get_audit(tx, id).await?;
Ok(Json(audit))
}
async fn list_audit(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(lq): Query<ListAuditLogQuery>,
) -> JsonResult<Vec<AuditLog>> {
let tx = user_db.begin(&authed).await?;
let rows = windmill_audit::list_audit(tx, w_id, pagination, lq).await?;
Ok(Json(rows))
}
backend/src/capture.rs → backend/windmill-api/src/capture.rs
+12 -2
View File
@@ -1,15 +1,25 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{
extract::{Extension, Path},
routing::{get, post, put},
Json, Router,
};
use hyper::StatusCode;
use windmill_common::{
error::{JsonResult, Result},
utils::{not_found_if_none, StripPath},
};
use crate::{
db::{UserDB, DB},
error::{JsonResult, Result},
users::Authed,
utils::{not_found_if_none, StripPath},
};
const KEEP_LAST: i64 = 8;
backend/src/db.rs → backend/windmill-api/src/db.rs
+5 -13
View File
@@ -6,23 +6,15 @@
* LICENSE-AGPL for a copy of the license.
*/
use crate::{error::Error, users::Authed};
use sqlx::{postgres::PgPoolOptions, Pool, Postgres, Transaction};
use std::time::Duration;
use sqlx::{Pool, Postgres, Transaction};
use windmill_common::error::Error;
use crate::users::Authed;
pub type DB = Pool<Postgres>;
pub async fn connect(database_url: &str, max_connections: u32) -> Result<DB, Error> {
PgPoolOptions::new()
.max_connections(max_connections)
.max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
.connect(database_url)
.await
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))
}
pub async fn migrate(db: &DB) -> Result<(), Error> {
match sqlx::migrate!("./migrations").run(db).await {
match sqlx::migrate!("../migrations").run(db).await {
Ok(_) => Ok(()),
Err(err) => Err(err),
}?;
backend/src/flows.rs → backend/windmill-api/src/flows.rs
+22 -218
View File
@@ -6,9 +6,6 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::HashMap;
use std::time::Duration;
use reqwest::Client;
use sql_builder::prelude::*;
@@ -17,18 +14,20 @@ use axum::{
routing::{get, post},
Json, Router,
};
use serde::{Deserialize, Serialize};
use sql_builder::SqlBuilder;
use sqlx::{FromRow, Postgres, Transaction};
use sqlx::{Postgres, Transaction};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{self, to_anyhow, Error, JsonResult, Result},
flows::{Flow, ListFlowQuery, NewFlow},
utils::{
http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
},
};
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{self, to_anyhow, Error, JsonResult, Result},
more_serde::{default_id, default_true, is_default},
scripts::{Schema, ScriptLang},
users::Authed,
utils::{http_get_from_hub, list_elems_from_hub, Pagination, StripPath},
};
pub fn workspaced_service() -> Router {
@@ -47,208 +46,6 @@ pub fn global_service() -> Router {
.route("/hub/get/:id", get(get_hub_flow_by_id))
}
#[derive(FromRow, Serialize)]
pub struct Flow {
pub workspace_id: String,
pub path: String,
pub summary: String,
pub description: String,
pub value: serde_json::Value,
pub edited_by: String,
pub edited_at: chrono::DateTime<chrono::Utc>,
pub archived: bool,
pub schema: Option<Schema>,
pub extra_perms: serde_json::Value,
}
#[derive(FromRow, Deserialize)]
pub struct NewFlow {
pub path: String,
pub summary: String,
pub description: String,
pub value: serde_json::Value,
pub schema: Option<Schema>,
}
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct FlowValue {
pub modules: Vec<FlowModule>,
#[serde(default)]
pub failure_module: Option<FlowModule>,
#[serde(default)]
#[serde(skip_serializing_if = "is_default")]
pub same_worker: bool,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct StopAfterIf {
pub expr: String,
pub skip_if_stopped: bool,
}
#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct Retry {
constant: ConstantDelay,
exponential: ExponentialDelay,
}
impl Retry {
/// Takes the number of previous retries and returns the interval until the next retry if any.
///
/// May return [`Duration::ZERO`] to retry immediately.
pub fn interval(&self, previous_attempts: u16) -> Option<Duration> {
let Self { constant, exponential } = self;
if previous_attempts < constant.attempts {
Some(Duration::from_secs(constant.seconds as u64))
} else if previous_attempts - constant.attempts < exponential.attempts {
let exp = previous_attempts.saturating_add(1) as u32;
let secs = exponential.multiplier * exponential.seconds.saturating_pow(exp);
Some(Duration::from_secs(secs as u64))
} else {
None
}
}
pub fn has_attempts(&self) -> bool {
self.constant.attempts != 0 || self.exponential.attempts != 0
}
pub fn max_attempts(&self) -> u16 {
self.constant
.attempts
.saturating_add(self.exponential.attempts)
}
pub fn max_interval(&self) -> Option<Duration> {
self.max_attempts()
.checked_sub(1)
.and_then(|p| self.interval(p))
}
}
#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct ConstantDelay {
pub attempts: u16,
pub seconds: u16,
}
/// multiplier * seconds ^ failures
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(default)]
pub struct ExponentialDelay {
pub attempts: u16,
pub multiplier: u16,
pub seconds: u16,
}
impl Default for ExponentialDelay {
fn default() -> Self {
Self { attempts: 0, multiplier: 1, seconds: 0 }
}
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct Suspend {
#[serde(skip_serializing_if = "Option::is_none")]
pub required_events: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub timeout: Option<u32>,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct FlowModule {
#[serde(default = "default_id")]
pub id: String,
#[serde(default)]
#[serde(alias = "input_transform")]
pub input_transforms: HashMap<String, InputTransform>,
pub value: FlowModuleValue,
pub stop_after_if: Option<StopAfterIf>,
pub summary: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub suspend: Option<Suspend>,
#[serde(skip_serializing_if = "Option::is_none")]
pub retry: Option<Retry>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sleep: Option<InputTransform>,
}
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum InputTransform {
Static { value: serde_json::Value },
Javascript { expr: String },
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchOneModules {
#[serde(skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
pub expr: String,
pub modules: Vec<FlowModule>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchAllModules {
#[serde(skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
pub modules: Vec<FlowModule>,
#[serde(default = "default_true")]
pub skip_failure: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum FlowModuleValue {
Script {
#[serde(default)]
#[serde(alias = "input_transform")]
input_transforms: HashMap<String, InputTransform>,
path: String,
},
ForloopFlow {
iterator: InputTransform,
modules: Vec<FlowModule>,
#[serde(default = "default_true")]
skip_failures: bool,
},
BranchOne {
branches: Vec<BranchOneModules>,
default: Vec<FlowModule>,
},
BranchAll {
branches: Vec<BranchAllModules>,
},
RawScript {
#[serde(default)]
#[serde(alias = "input_transform")]
input_transforms: HashMap<String, InputTransform>,
content: String,
path: Option<String>,
language: ScriptLang,
},
Identity,
}
#[derive(Deserialize)]
pub struct ListFlowQuery {
pub path_start: Option<String>,
pub path_exact: Option<String>,
pub edited_by: Option<String>,
pub show_archived: Option<bool>,
pub order_by: Option<String>,
pub order_desc: Option<bool>,
}
async fn list_flows(
authed: Authed,
Extension(user_db): Extension<UserDB>,
@@ -256,7 +53,7 @@ async fn list_flows(
Query(pagination): Query<Pagination>,
Query(lq): Query<ListFlowQuery>,
) -> JsonResult<Vec<Flow>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let (per_page, offset) = paginate(pagination);
let mut sqlb = SqlBuilder::select_from("flow as o")
.fields(&[
@@ -428,7 +225,7 @@ async fn update_flow(
)
.fetch_optional(&mut tx)
.await?;
crate::utils::not_found_if_none(flow, "Flow", flow_path)?;
not_found_if_none(flow, "Flow", flow_path)?;
audit_log(
&mut tx,
@@ -467,7 +264,7 @@ async fn get_flow_by_path(
.await?;
tx.commit().await?;
let flow = crate::utils::not_found_if_none(flow_o, "Flow", path)?;
let flow = not_found_if_none(flow_o, "Flow", path)?;
Ok(Json(flow))
}
@@ -524,8 +321,15 @@ async fn archive_flow_by_path(
#[cfg(test)]
mod tests {
// Note this useful idiom: importing names from outer (for mod tests) scope.
use super::*;
use std::{collections::HashMap, time::Duration};
use windmill_common::{
flows::{
ConstantDelay, ExponentialDelay, FlowModule, FlowModuleValue, FlowValue,
InputTransform, Retry, StopAfterIf,
},
scripts,
};
const SECOND: Duration = Duration::from_secs(1);
@@ -556,7 +360,7 @@ mod tests {
value: FlowModuleValue::RawScript {
input_transforms: HashMap::new(),
content: "test".to_string(),
language: crate::scripts::ScriptLang::Deno,
language: scripts::ScriptLang::Deno,
path: None,
},
stop_after_if: Some(StopAfterIf {
backend/src/granular_acls.rs → backend/windmill-api/src/granular_acls.rs
+8 -9
View File
@@ -6,12 +6,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
db::UserDB,
error::{Error, JsonResult, Result},
users::Authed,
utils::StripPath,
};
use crate::{db::UserDB, users::Authed};
use axum::{
extract::{Extension, Path},
routing::{get, post},
@@ -19,6 +14,10 @@ use axum::{
};
use serde::{Deserialize, Serialize};
use windmill_common::{
error::{Error, JsonResult, Result},
utils::{not_found_if_none, StripPath},
};
pub fn workspaced_service() -> Router {
Router::new()
@@ -56,7 +55,7 @@ async fn add_granular_acl(
.fetch_optional(&mut tx)
.await?;
let _ = crate::utils::not_found_if_none(obj_o, &kind, &path)?;
let _ = not_found_if_none(obj_o, &kind, &path)?;
tx.commit().await?;
Ok("Successfully modified granular acl".to_string())
@@ -85,7 +84,7 @@ async fn remove_granular_acl(
.fetch_optional(&mut tx)
.await?;
let _ = crate::utils::not_found_if_none(obj_o, &kind, &path)?;
let _ = not_found_if_none(obj_o, &kind, &path)?;
tx.commit().await?;
Ok("Successfully removed granular acl".to_string())
@@ -112,7 +111,7 @@ async fn get_granular_acls(
.fetch_optional(&mut tx)
.await?;
let obj = crate::utils::not_found_if_none(obj_o, &kind, &path)?;
let obj = not_found_if_none(obj_o, &kind, &path)?;
tx.commit().await?;
Ok(Json(obj))
backend/src/groups.rs → backend/windmill-api/src/groups.rs
+13 -14
View File
@@ -7,17 +7,20 @@
*/
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
users::{owner_to_token_owner, Authed},
utils::Pagination,
users::Authed,
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{Error, JsonResult, Result},
users::owner_to_token_owner,
utils::{not_found_if_none, paginate, Pagination},
};
use serde::{Deserialize, Serialize};
use sqlx::{FromRow, Postgres, Transaction};
@@ -72,7 +75,7 @@ async fn list_groups(
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
) -> JsonResult<Vec<Group>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let (per_page, offset) = paginate(pagination);
let rows = sqlx::query_as!(
Group,
@@ -158,11 +161,7 @@ async fn get_group(
) -> JsonResult<GroupInfo> {
let mut tx = user_db.begin(&authed).await?;
let group = crate::utils::not_found_if_none(
get_group_opt(&mut tx, &w_id, &name).await?,
"Group",
&name,
)?;
let group = not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
let members = sqlx::query_scalar!(
"SELECT usr.username
@@ -191,7 +190,7 @@ async fn delete_group(
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
sqlx::query!(
"DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2",
@@ -229,7 +228,7 @@ async fn update_group(
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
sqlx::query_as!(
Group,
@@ -263,7 +262,7 @@ async fn add_user(
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
sqlx::query_as!(
Group,
@@ -297,7 +296,7 @@ async fn remove_user(
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
if &name == "all" {
return Err(Error::BadRequest(format!("Cannot delete users from all")));
}
backend/src/jobs.rs → backend/windmill-api/src/jobs.rs
+642 -1326
View File
File diff suppressed because it is too large Load Diff
backend/src/lib.rs → backend/windmill-api/src/lib.rs
+13 -152
View File
@@ -6,41 +6,31 @@
* LICENSE-AGPL for a copy of the license.
*/
use anyhow::Context;
use argon2::Argon2;
use axum::{handler::Handler, middleware::from_extractor, routing::get, Extension, Router};
use db::DB;
use futures::FutureExt;
use git_version::git_version;
use std::{net::SocketAddr, sync::Arc};
use tower::ServiceBuilder;
use tower_cookies::CookieManagerLayer;
use tower_http::trace::TraceLayer;
use windmill_common::{error::to_anyhow, utils::rd_string};
extern crate magic_crypt;
extern crate dotenv;
use crate::{
db::UserDB,
oauth2::{build_oauth_clients, SlackVerifier},
tracing_init::{MyMakeSpan, MyOnResponse},
users::Authed,
};
mod audit;
mod capture;
mod client;
mod db;
mod error;
mod external_ip;
mod flows;
mod granular_acls;
mod groups;
mod jobs;
mod js_eval;
mod more_serde;
pub mod jobs;
mod oauth2;
mod parser;
mod parser_go;
mod parser_go_ast;
mod parser_go_scanner;
mod parser_go_token;
mod parser_py;
mod parser_ts;
mod resources;
mod schedule;
mod scripts;
@@ -49,51 +39,17 @@ mod tracing_init;
mod users;
mod utils;
mod variables;
mod worker;
mod worker_flow;
mod worker_ping;
mod workspaces;
use error::Error;
use crate::{
db::UserDB,
error::to_anyhow,
oauth2::{build_oauth_clients, SlackVerifier},
tracing_init::{MyMakeSpan, MyOnResponse},
utils::rd_string,
};
pub use crate::tracing_init::initialize_tracing;
pub use crate::worker::WorkerConfig;
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
pub const DEFAULT_NUM_WORKERS: usize = 3;
pub const DEFAULT_TIMEOUT: i32 = 300;
pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
pub const DEFAULT_MAX_CONNECTIONS: u32 = 100;
pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
db::migrate(db).await?;
Ok(())
}
pub async fn connect_db() -> anyhow::Result<DB> {
let database_url = std::env::var("DATABASE_URL")
.map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
Err(_) => DEFAULT_MAX_CONNECTIONS,
};
Ok(db::connect(&database_url, max_connections).await?)
}
struct BaseUrl(String);
struct IsSecure(bool);
struct CloudHosted(bool);
pub use users::delete_expired_items_perdiodically;
pub async fn run_server(
db: DB,
addr: SocketAddr,
@@ -168,7 +124,7 @@ pub async fn run_server(
.nest("/scripts", scripts::global_service())
.nest("/flows", flows::global_service())
.nest("/schedules", schedule::global_service())
.route_layer(from_extractor::<users::Authed>())
.route_layer(from_extractor::<Authed>())
.route_layer(from_extractor::<users::Tokened>())
.nest("/w/:workspace_id/jobs", jobs::global_service())
.nest("/w/:workspace_id/capture", capture::global_service())
@@ -202,64 +158,6 @@ pub async fn run_server(
Ok(())
}
pub fn monitor_db(db: &DB, timeout: i32, rx: tokio::sync::broadcast::Receiver<()>) {
let db1 = db.clone();
let db2 = db.clone();
let rx2 = rx.resubscribe();
tokio::spawn(async move { worker::handle_zombie_jobs_periodically(&db1, timeout, rx).await });
tokio::spawn(async move { users::delete_expired_items_perdiodically(&db2, rx2).await });
}
pub async fn run_workers(
db: DB,
addr: SocketAddr,
timeout: i32,
num_workers: i32,
sleep_queue: u64,
worker_config: WorkerConfig,
rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<()> {
let instance_name = rd_string(5);
let monitor = tokio_metrics::TaskMonitor::new();
let ip = external_ip::get_ip().await.unwrap_or_else(|e| {
tracing::warn!(error = e.to_string(), "failed to get external IP");
"unretrievable IP".to_string()
});
let mut handles = Vec::with_capacity(num_workers as usize);
for i in 1..(num_workers + 1) {
let db1 = db.clone();
let instance_name = instance_name.clone();
let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
let ip = ip.clone();
let rx = rx.resubscribe();
let worker_config = worker_config.clone();
handles.push(tokio::spawn(monitor.instrument(async move {
tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
worker::run_worker(
&db1,
timeout,
&instance_name,
worker_name,
i as u64,
num_workers as u64,
&ip,
sleep_queue,
worker_config,
rx,
)
.await
})));
}
futures::future::try_join_all(handles).await?;
Ok(())
}
async fn git_v() -> &'static str {
GIT_VERSION
}
@@ -267,44 +165,7 @@ async fn git_v() -> &'static str {
async fn openapi() -> &'static str {
include_str!("../openapi.yaml")
}
pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
use std::io;
use tokio::signal::unix::SignalKind;
async fn terminate() -> io::Result<()> {
tokio::signal::unix::signal(SignalKind::terminate())?
.recv()
.await;
Ok(())
}
tokio::select! {
_ = terminate() => {},
_ = tokio::signal::ctrl_c() => {},
}
println!("signal received, starting graceful shutdown");
let _ = tx.send(());
pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
db::migrate(db).await?;
Ok(())
}
pub async fn serve_metrics(
addr: SocketAddr,
mut rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<(), hyper::Error> {
axum::Server::bind(&addr)
.serve(
Router::new()
.route("/metrics", get(metrics))
.into_make_service(),
)
.with_graceful_shutdown(rx.recv().map(drop))
.await
}
async fn metrics() -> Result<String, Error> {
let metric_families = prometheus::gather();
Ok(prometheus::TextEncoder::new()
.encode_to_string(&metric_families)
.map_err(anyhow::Error::from)?)
}
backend/windmill-api/src/main.rs
+70
View File
@@ -0,0 +1,70 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::net::SocketAddr;
use anyhow::Ok;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
windmill_common::tracing_init::initialize_tracing();
let db = windmill_common::connect_db().await?;
let num_workers = std::env::var("NUM_WORKERS")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32);
let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
.ok()
.map(|s| {
s.parse::<bool>()
.map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
.or_else(|_| s.parse::<SocketAddr>().map(Some))
})
.transpose()?
.flatten();
let server_mode = !std::env::var("DISABLE_SERVER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
if server_mode {
windmill_api::migrate_db(&db).await?;
}
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
let shutdown_signal = windmill_common::shutdown_signal(tx);
let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
if server_mode || num_workers > 0 {
let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
let server_f = async {
if server_mode {
windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?;
}
Ok(()) as anyhow::Result<()>
};
let metrics_f = async {
match metrics_addr {
Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
.await
.map_err(anyhow::Error::from),
None => Ok(()),
}
};
futures::try_join!(shutdown_signal, server_f, metrics_f)?;
}
Ok(())
}
backend/src/oauth2.rs → backend/windmill-api/src/oauth2.rs
+23 -17
View File
@@ -1,3 +1,11 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::{collections::HashMap, fmt::Debug};
use std::sync::Arc;
@@ -10,6 +18,7 @@ use axum::{
routing::{get, post},
Json, Router,
};
use hmac::Mac;
use hyper::StatusCode;
use itertools::Itertools;
@@ -19,29 +28,24 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sqlx::{Postgres, Transaction};
use tokio::{fs::File, io::AsyncReadExt};
use tower_cookies::{Cookie, Cookies};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::utils::{not_found_if_none, now_from_db};
use crate::utils::now_from_db;
use crate::users::Authed;
use crate::IsSecure;
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{self, to_anyhow, Error, Result},
jobs,
jobs::{get_latest_hash_for_path, JobPayload},
users::Authed,
utils::not_found_if_none,
variables::{build_crypt, encrypt},
workspaces::WorkspaceSettings,
BaseUrl,
};
use windmill_common::error::{self, to_anyhow, Error, Result};
use windmill_common::oauth2::*;
use windmill_queue::JobPayload;
use std::str;
use hmac::{Hmac, Mac};
use sha2::Sha256;
pub type HmacSha256 = Hmac<Sha256>;
pub fn global_service() -> Router {
Router::new()
.route("/login/:client", get(login))
@@ -97,10 +101,11 @@ pub struct AllClients {
pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
let connect_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
"../oauth_connect.json"
"../../oauth_connect.json"
))?;
let login_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
"../../oauth_login.json"
))?;
let login_configs =
serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!("../oauth_login.json"))?;
let mut content = String::new();
let path = "./oauth.json";
@@ -665,7 +670,8 @@ async fn slack_command(
if let Some(settings) = settings {
if let Some(script) = &settings.slack_command_script {
let script_hash =
get_latest_hash_for_path(&mut tx, &settings.workspace_id, script).await?;
windmill_common::get_latest_hash_for_path(&mut tx, &settings.workspace_id, script)
.await?;
let mut map = serde_json::Map::new();
map.insert("text".to_string(), serde_json::Value::String(form.text));
map.insert(
@@ -673,7 +679,7 @@ async fn slack_command(
serde_json::Value::String(form.response_url),
);
let (uuid, tx) = jobs::push(
let (uuid, tx) = windmill_queue::push(
tx,
&settings.workspace_id,
JobPayload::ScriptHash { hash: script_hash, path: script.to_owned() },
backend/src/resources.rs → backend/windmill-api/src/resources.rs
+10 -8
View File
@@ -7,11 +7,8 @@
*/
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
users::Authed,
utils::{require_admin, Pagination, StripPath},
};
use axum::{
extract::{Extension, Path, Query},
@@ -22,6 +19,11 @@ use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use sql_builder::{bind::Bind, SqlBuilder};
use sqlx::FromRow;
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{Error, JsonResult, Result},
utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
};
pub fn workspaced_service() -> Router {
Router::new()
@@ -99,7 +101,7 @@ async fn list_resources(
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<Resource>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let (per_page, offset) = paginate(pagination);
let mut sqlb = SqlBuilder::select_from("resource")
.fields(&[
@@ -150,7 +152,7 @@ async fn get_resource(
.await?;
tx.commit().await?;
let resource = crate::utils::not_found_if_none(resource_o, "Resource", path)?;
let resource = not_found_if_none(resource_o, "Resource", path)?;
Ok(Json(resource))
}
@@ -190,7 +192,7 @@ async fn get_resource_value(
.await?;
tx.commit().await?;
let value = crate::utils::not_found_if_none(value_o, "Resource", path)?;
let value = not_found_if_none(value_o, "Resource", path)?;
Ok(Json(value))
}
@@ -294,7 +296,7 @@ async fn update_resource(
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
let npath = crate::utils::not_found_if_none(npath_o, "Resource", path)?;
let npath = not_found_if_none(npath_o, "Resource", path)?;
audit_log(
&mut tx,
@@ -360,7 +362,7 @@ async fn get_resource_type(
.await?;
tx.commit().await?;
let resource_type = crate::utils::not_found_if_none(resource_type_o, "ResourceType", name)?;
let resource_type = not_found_if_none(resource_type_o, "ResourceType", name)?;
Ok(Json(resource_type))
}
backend/windmill-api/src/schedule.rs
+126
View File
@@ -0,0 +1,126 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
db::{UserDB, DB},
users::Authed,
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use chrono::DateTime;
use windmill_common::{
error::{JsonResult, Result},
utils::{not_found_if_none, Pagination, StripPath},
};
use windmill_queue::{
self,
schedule::{EditSchedule, NewSchedule, PreviewPayload, Schedule, SetEnabled},
};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_schedule))
.route("/get/*path", get(get_schedule))
.route("/exists/*path", get(exists_schedule))
.route("/create", post(create_schedule))
.route("/update/*path", post(edit_schedule))
.route("/delete/*path", delete(delete_schedule))
.route("/setenabled/*path", post(set_enabled))
}
pub fn global_service() -> Router {
Router::new().route("/preview", post(preview_schedule))
}
async fn create_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(ns): Json<NewSchedule>,
) -> Result<String> {
let tx = user_db.begin(&authed).await?;
let res = windmill_queue::schedule::create_schedule(tx, w_id, ns, &authed.username).await?;
Ok(res)
}
async fn edit_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(es): Json<EditSchedule>,
) -> Result<String> {
let tx = user_db.begin(&authed).await?;
let res = windmill_queue::schedule::edit_schedule(tx, w_id, path, es, &authed.username).await?;
Ok(res)
}
async fn list_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
) -> JsonResult<Vec<Schedule>> {
let tx = user_db.begin(&authed).await?;
let res = windmill_queue::schedule::list_schedule(tx, w_id, pagination).await?;
Ok(Json(res))
}
async fn get_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Schedule> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let schedule_o = windmill_queue::schedule::get_schedule_opt(&mut tx, &w_id, path).await?;
let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
tx.commit().await?;
Ok(Json(schedule))
}
async fn exists_schedule(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<bool> {
let mut tx = db.begin().await?;
let res = windmill_queue::schedule::exists_schedule(&mut tx, w_id, path).await?;
tx.commit().await?;
Ok(Json(res))
}
pub async fn preview_schedule(
Json(payload): Json<PreviewPayload>,
) -> JsonResult<Vec<DateTime<chrono::Utc>>> {
Ok(Json(windmill_queue::schedule::preview_schedule(payload)?))
}
pub async fn set_enabled(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<SetEnabled>,
) -> Result<String> {
let tx = user_db.begin(&authed).await?;
let res =
windmill_queue::schedule::set_enabled(tx, w_id, path, payload, &authed.username).await?;
Ok(res)
}
async fn delete_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let tx = user_db.begin(&authed).await?;
let res = windmill_queue::schedule::delete_schedule(tx, w_id, path, &authed.username).await?;
Ok(res)
}
backend/src/scripts.rs → backend/windmill-api/src/scripts.rs
+47 -218
View File
@@ -7,16 +7,12 @@
*/
use reqwest::Client;
use serde::Deserializer;
use sql_builder::prelude::*;
use windmill_audit::{audit_log, ActionKind};
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{to_anyhow, Error, JsonResult, Result},
jobs, parser, parser_go, parser_py, parser_ts,
users::{owner_to_token_owner, truncate_token, Authed, Tokened},
utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath},
users::{truncate_token, Authed, Tokened},
};
use axum::{
extract::{Extension, Host, Path, Query},
@@ -24,15 +20,25 @@ use axum::{
Json, Router,
};
use hyper::StatusCode;
use serde::{de::Error as _, ser::SerializeSeq, Deserialize, Serialize};
use serde_json::{json, to_string_pretty};
use serde::Serialize;
use serde_json::json;
use sql_builder::SqlBuilder;
use sqlx::{FromRow, Postgres, Transaction};
use std::{
collections::hash_map::DefaultHasher,
fmt::Display,
hash::{Hash, Hasher},
};
use windmill_common::{
error::{Error, JsonResult, Result},
scripts::{
to_i64, HubScript, ListScriptQuery, NewScript, Script, ScriptHash, ScriptKind, ScriptLang,
},
users::owner_to_token_owner,
utils::{
list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
},
};
use windmill_queue;
const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
@@ -63,145 +69,6 @@ pub fn workspaced_service() -> Router {
.route("/raw/h/:hash", get(raw_script_by_hash))
.route("/deployment_status/h/:hash", get(get_deployment_status))
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum ScriptLang {
Deno,
Python3,
Go,
}
impl ScriptLang {
pub fn as_str(&self) -> &'static str {
match self {
ScriptLang::Deno => "deno",
ScriptLang::Python3 => "python3",
ScriptLang::Go => "go",
}
}
}
#[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)]
#[sqlx(transparent)]
pub struct ScriptHash(pub i64);
#[derive(sqlx::Type, PartialEq)]
#[sqlx(transparent)]
pub struct ScriptHashes(Vec<i64>);
impl Display for ScriptHash {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", to_hex_string(&self.0))
}
}
impl Serialize for ScriptHash {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(to_hex_string(&self.0).as_str())
}
}
impl<'de> Deserialize<'de> for ScriptHash {
fn deserialize<D>(deserializer: D) -> std::result::Result<ScriptHash, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
let i = to_i64(&s).map_err(|e| D::Error::custom(format!("{}", e)))?;
Ok(ScriptHash(i))
}
}
impl Serialize for ScriptHashes {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for element in &self.0 {
seq.serialize_element(&ScriptHash(*element))?;
}
seq.end()
}
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, Hash)]
#[sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum ScriptKind {
Trigger,
Failure,
Script,
Approval,
}
#[derive(FromRow, Serialize)]
pub struct Script {
pub workspace_id: String,
pub hash: ScriptHash,
pub path: String,
pub parent_hashes: Option<ScriptHashes>,
pub summary: String,
pub description: String,
pub content: String,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub archived: bool,
pub schema: Option<Schema>,
pub deleted: bool,
pub is_template: bool,
pub extra_perms: serde_json::Value,
pub lock: Option<String>,
pub lock_error_logs: Option<String>,
pub language: ScriptLang,
pub kind: ScriptKind,
}
#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
#[sqlx(transparent)]
#[serde(transparent)]
pub struct Schema(pub serde_json::Value);
impl Hash for Schema {
fn hash<H: Hasher>(&self, state: &mut H) {
if let Ok(s) = to_string_pretty(&self.0) {
s.hash(state);
}
}
}
#[derive(Serialize, Deserialize, Hash)]
pub struct NewScript {
pub path: String,
pub parent_hash: Option<ScriptHash>,
pub summary: String,
pub description: String,
pub content: String,
pub schema: Option<Schema>,
pub is_template: Option<bool>,
pub lock: Option<Vec<String>>,
pub language: ScriptLang,
pub kind: Option<ScriptKind>,
}
#[derive(Deserialize)]
pub struct ListScriptQuery {
pub path_start: Option<String>,
pub path_exact: Option<String>,
pub created_by: Option<String>,
pub first_parent_hash: Option<ScriptHash>,
pub last_parent_hash: Option<ScriptHash>,
pub parent_hash: Option<ScriptHash>,
pub show_archived: Option<bool>,
pub order_by: Option<String>,
pub order_desc: Option<bool>,
pub is_template: Option<bool>,
pub kind: Option<String>,
}
async fn list_scripts(
authed: Authed,
Extension(user_db): Extension<UserDB>,
@@ -209,7 +76,7 @@ async fn list_scripts(
Query(pagination): Query<Pagination>,
Query(lq): Query<ListScriptQuery>,
) -> JsonResult<Vec<Script>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let (per_page, offset) = paginate(pagination);
let mut sqlb = SqlBuilder::select_from("script as o")
.fields(&[
@@ -443,13 +310,15 @@ async fn create_script(
let mut tx = if ns.lock.is_none() && ns.language != ScriptLang::Deno {
let dependencies = match ns.language {
ScriptLang::Python3 => parser_py::parse_python_imports(&ns.content)?.join("\n"),
ScriptLang::Python3 => {
windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
}
_ => ns.content,
};
let (_, tx) = jobs::push(
let (_, tx) = windmill_queue::push(
tx,
&w_id,
jobs::JobPayload::Dependencies { hash, dependencies, language: ns.language },
windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
None,
&authed.username,
owner_to_token_owner(&authed.username, false),
@@ -508,63 +377,37 @@ async fn create_script(
}
pub async fn get_hub_script_by_path(
Authed { email, username, .. }: Authed,
authed: Authed,
Path(path): Path<StripPath>,
Extension(http_client): Extension<Client>,
Host(host): Host,
) -> Result<String> {
let path = path
.to_path()
.strip_prefix("hub/")
.ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
let content = http_get_from_hub(
windmill_common::scripts::get_hub_script_by_path(
authed.email,
authed.username,
path,
http_client,
&format!("https://hub.windmill.dev/raw/{path}.ts"),
email,
username,
host,
true,
)
.await?
.text()
.await
.map_err(to_anyhow)?;
Ok(content)
}
#[derive(Deserialize, Serialize)]
pub struct HubScript {
pub content: String,
pub lockfile: Option<String>,
pub language: ScriptLang,
pub schema: Option<serde_json::Value>,
}
pub async fn get_full_hub_script_by_path(
Authed { email, username, .. }: Authed,
Authed { username, email, .. }: Authed,
Path(path): Path<StripPath>,
Extension(http_client): Extension<Client>,
Host(host): Host,
) -> JsonResult<HubScript> {
let path = path
.to_path()
.strip_prefix("hub/")
.ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
let value = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/raw2/{path}"),
email,
username,
host,
true,
)
.await?
.json::<HubScript>()
.await
.map_err(to_anyhow)?;
Ok(Json(value))
Ok(Json(
windmill_common::scripts::get_full_hub_script_by_path(
email,
username,
path,
http_client,
host,
)
.await?,
))
}
async fn get_script_by_path(
@@ -586,7 +429,7 @@ async fn get_script_by_path(
.await?;
tx.commit().await?;
let script = crate::utils::not_found_if_none(script_o, "Script", path)?;
let script = not_found_if_none(script_o, "Script", path)?;
Ok(Json(script))
}
@@ -612,7 +455,7 @@ async fn raw_script_by_path(
.await?;
tx.commit().await?;
let content = crate::utils::not_found_if_none(content_o, "Script", path)?;
let content = not_found_if_none(content_o, "Script", path)?;
Ok(content)
}
@@ -650,7 +493,7 @@ async fn get_script_by_hash_internal<'c>(
.fetch_optional(db)
.await?;
let script = crate::utils::not_found_if_none(script_o, "Script", hash.to_string())?;
let script = not_found_if_none(script_o, "Script", hash.to_string())?;
Ok(script)
}
@@ -702,7 +545,7 @@ async fn get_deployment_status(
.fetch_optional(&mut tx)
.await?;
let status = crate::utils::not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?;
let status = not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?;
tx.commit().await?;
Ok(Json(status))
@@ -806,31 +649,17 @@ async fn delete_script_by_hash(
async fn parse_python_code_to_jsonschema(
Json(code): Json<String>,
) -> JsonResult<parser::MainArgSignature> {
parser_py::parse_python_signature(&code).map(Json)
) -> JsonResult<windmill_parser::MainArgSignature> {
windmill_parser_py::parse_python_signature(&code).map(Json)
}
async fn parse_deno_code_to_jsonschema(
Json(code): Json<String>,
) -> JsonResult<parser::MainArgSignature> {
parser_ts::parse_deno_signature(&code).map(Json)
) -> JsonResult<windmill_parser::MainArgSignature> {
windmill_parser_ts::parse_deno_signature(&code).map(Json)
}
async fn parse_go_code_to_jsonschema(
Json(code): Json<String>,
) -> JsonResult<parser::MainArgSignature> {
parser_go::parse_go_sig(&code).map(Json)
}
pub fn to_i64(s: &str) -> Result<i64> {
let v = hex::decode(s)?;
let nb: u64 = u64::from_be_bytes(
v[0..8]
.try_into()
.map_err(|_| hex::FromHexError::InvalidStringLength)?,
);
Ok(nb as i64)
}
pub fn to_hex_string(i: &i64) -> String {
hex::encode(i.to_be_bytes())
) -> JsonResult<windmill_parser::MainArgSignature> {
windmill_parser_go::parse_go_sig(&code).map(Json)
}
backend/src/static_assets.rs → backend/windmill-api/src/static_assets.rs
+1 -1
View File
@@ -22,7 +22,7 @@ pub async fn static_handler(uri: Uri) -> impl IntoResponse {
}
#[derive(RustEmbed)]
#[folder = "../frontend/build/"]
#[folder = "../../frontend/build/"]
struct Asset;
pub struct StaticFile<T>(pub T);
backend/windmill-api/src/tracing_init.rs
+45
View File
@@ -0,0 +1,45 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use ::tracing::{field, Span};
use hyper::Response;
use tower_http::trace::{MakeSpan, OnResponse};
#[derive(Clone)]
pub struct MyOnResponse {}
impl<B> OnResponse<B> for MyOnResponse {
fn on_response(
self,
response: &Response<B>,
latency: std::time::Duration,
_span: &tracing::Span,
) {
tracing::info!(
latency = latency.as_millis(),
status = response.status().as_u16(),
"response"
)
}
}
#[derive(Clone)]
pub struct MyMakeSpan {}
impl<B> MakeSpan<B> for MyMakeSpan {
fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
tracing::info_span!(
"request",
method = %request.method(),
uri = %request.uri(),
username = field::Empty,
workspace_id = field::Empty,
email = field::Empty,
)
}
}
backend/windmill-api/src/utils.rs
+30
View File
@@ -0,0 +1,30 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use sqlx::{Postgres, Transaction};
use windmill_common::error::{self, Error};
pub async fn require_super_admin<'c>(
db: &mut Transaction<'c, Postgres>,
email: Option<String>,
) -> error::Result<()> {
let is_admin = sqlx::query_scalar!(
"SELECT super_admin FROM password WHERE email = $1",
email.as_ref()
)
.fetch_one(db)
.await
.map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
if !is_admin {
Err(Error::NotAuthorized(
"This endpoint require caller to be a super admin".to_owned(),
))
} else {
Ok(())
}
}
backend/src/variables.rs → backend/windmill-api/src/variables.rs
+26 -123
View File
@@ -9,25 +9,36 @@
use std::sync::Arc;
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
oauth2::{AllClients, _refresh_token},
users::Authed,
utils::StripPath,
BaseUrl,
};
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use hyper::StatusCode;
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{Error, JsonResult, Result},
utils::{not_found_if_none, StripPath},
variables::{get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable},
};
use magic_crypt::{MagicCrypt256, MagicCryptTrait};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use sqlx::{FromRow, Postgres, Transaction};
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
pub fn workspaced_service() -> Router {
Router::new()
@@ -40,122 +51,6 @@ pub fn workspaced_service() -> Router {
.route("/create", post(create_variable))
}
#[derive(Serialize, Clone)]
pub struct ContextualVariable {
pub name: String,
pub value: String,
pub description: String,
}
#[derive(Serialize, Deserialize, FromRow)]
pub struct ListableVariable {
pub workspace_id: String,
pub path: String,
pub value: Option<String>,
pub is_secret: bool,
pub description: String,
pub extra_perms: serde_json::Value,
pub account: Option<i32>,
pub is_oauth: bool,
pub is_expired: Option<bool>,
}
#[derive(Deserialize)]
pub struct CreateVariable {
pub path: String,
pub value: String,
pub is_secret: bool,
pub description: String,
pub account: Option<i32>,
pub is_oauth: Option<bool>,
}
#[derive(Deserialize)]
struct EditVariable {
path: Option<String>,
value: Option<String>,
is_secret: Option<bool>,
description: Option<String>,
}
pub fn get_reserved_variables(
w_id: &str,
token: &str,
email: &str,
username: &str,
job_id: &str,
permissioned_as: &str,
base_url: &str,
path: Option<String>,
flow_id: Option<String>,
flow_path: Option<String>,
schedule_path: Option<String>,
) -> [ContextualVariable; 11] {
[
ContextualVariable {
name: "WM_WORKSPACE".to_string(),
value: w_id.to_string(),
description: "Workspace id of the current script".to_string(),
},
ContextualVariable {
name: "WM_TOKEN".to_string(),
value: token.to_string(),
description: "Token ephemeral to the current script with equal permission to the \
permission of the run (Usable as a bearer token)"
.to_string(),
},
ContextualVariable {
name: "WM_EMAIL".to_string(),
value: email.to_string(),
description: "Email of the user that executed the current script".to_string(),
},
ContextualVariable {
name: "WM_USERNAME".to_string(),
value: username.to_string(),
description: "Username of the user that executed the current script".to_string(),
},
ContextualVariable {
name: "WM_BASE_URL".to_string(),
value: base_url.to_string(),
description: "base url of this instance".to_string(),
},
ContextualVariable {
name: "WM_JOB_ID".to_string(),
value: job_id.to_string(),
description: "Job id of the current script".to_string(),
},
ContextualVariable {
name: "WM_JOB_PATH".to_string(),
value: path.unwrap_or_else(|| "".to_string()),
description: "Path of the script or flow being run if any".to_string(),
},
ContextualVariable {
name: "WM_FLOW_JOB_ID".to_string(),
value: flow_id.unwrap_or_else(|| "".to_string()),
description: "Job id of the encapsulating flow if the job is a flow step".to_string(),
},
ContextualVariable {
name: "WM_FLOW_PATH".to_string(),
value: flow_path.unwrap_or_else(|| "".to_string()),
description: "Path of the encapsulating flow if the job is a flow step".to_string(),
},
ContextualVariable {
name: "WM_SCHEDULE_PATH".to_string(),
value: schedule_path.unwrap_or_else(|| "".to_string()),
description: "Path of the schedule if the job of the step or encapsulating step has \
been triggered by a schedule"
.to_string(),
},
ContextualVariable {
name: "WM_PERMISSIONED_AS".to_string(),
value: permissioned_as.to_string(),
description: "Fully Qualified (u/g) owner name of executor of the job".to_string(),
},
]
}
async fn list_contextual_variables(
Path(w_id): Path<String>,
Extension(base_url): Extension<Arc<BaseUrl>>,
@@ -229,7 +124,7 @@ async fn get_variable(
.fetch_optional(&mut tx)
.await?;
let variable = crate::utils::not_found_if_none(variable_o, "Variable", &path)?;
let variable = not_found_if_none(variable_o, "Variable", &path)?;
let decrypt_secret = q.decrypt_secret.unwrap_or(true);
@@ -376,6 +271,14 @@ async fn delete_variable(
Ok(format!("variable {} deleted", path))
}
#[derive(Deserialize)]
struct EditVariable {
path: Option<String>,
value: Option<String>,
is_secret: Option<bool>,
description: Option<String>,
}
async fn update_variable(
authed: Authed,
Extension(user_db): Extension<UserDB>,
@@ -432,7 +335,7 @@ async fn update_variable(
let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
let npath = crate::utils::not_found_if_none(npath_o, "Variable", path)?;
let npath = not_found_if_none(npath_o, "Variable", path)?;
audit_log(
&mut tx,
backend/src/worker_ping.rs → backend/windmill-api/src/worker_ping.rs
+6 -2
View File
@@ -6,7 +6,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use crate::{db::UserDB, error::JsonResult, users::Authed, utils::Pagination};
use crate::{db::UserDB, users::Authed};
use axum::{
extract::{Extension, Query},
routing::get,
@@ -15,6 +15,10 @@ use axum::{
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use windmill_common::{
error::JsonResult,
utils::{paginate, Pagination},
};
pub fn global_service() -> Router {
Router::new().route("/list", get(list_worker_pings))
@@ -37,7 +41,7 @@ async fn list_worker_pings(
) -> JsonResult<Vec<WorkerPing>> {
let mut tx = user_db.begin(&authed).await?;
let (per_page, offset) = crate::utils::paginate(pagination);
let (per_page, offset) = paginate(pagination);
let rows = sqlx::query_as!(
WorkerPing,
backend/src/workspaces.rs → backend/windmill-api/src/workspaces.rs
+12 -8
View File
@@ -7,23 +7,27 @@
*/
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
flows::Flow,
resources::{Resource, ResourceType},
scripts::{Schema, Script, ScriptLang},
users::{Authed, WorkspaceInvite},
utils::{require_admin, require_super_admin, Pagination},
variables::ListableVariable,
utils::require_super_admin,
};
use axum::{
body::StreamBody,
extract::{Extension, Path, Query},
headers,
response::IntoResponse,
routing::{delete, get, post},
Json, Router,
};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{Error, JsonResult, Result},
flows::Flow,
scripts::{Schema, Script, ScriptLang},
utils::{paginate, rd_string, require_admin, Pagination},
variables::ListableVariable,
};
use hyper::{header, StatusCode};
use serde::{Deserialize, Serialize};
@@ -246,7 +250,7 @@ async fn list_workspaces_as_super_admin(
) -> JsonResult<Vec<Workspace>> {
let mut tx = user_db.begin(&authed).await?;
require_super_admin(&mut tx, email).await?;
let (per_page, offset) = crate::utils::paginate(pagination);
let (per_page, offset) = paginate(pagination);
let workspaces = sqlx::query_as!(
Workspace,
@@ -309,7 +313,7 @@ async fn create_workspace(
)
.execute(&mut tx)
.await?;
let key = crate::utils::rd_string(64);
let key = rd_string(64);
sqlx::query!(
"INSERT INTO workspace_key
(workspace_id, kind, key)
backend/windmill-audit/Cargo.toml
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "windmill-audit"
version = "0.1.0"
edition = "2021"
[lib]
name = "windmill_audit"
path = "./src/lib.rs"
[dependencies]
serde.workspace = true
sql-builder.workspace = true
sqlx.workspace = true
chrono.workspace = true
serde_json.workspace = true
tracing.workspace = true
windmill-common = { workspace = true, features = ["axum"] }
backend/src/audit.rs → backend/windmill-audit/src/lib.rs
+13 -34
View File
@@ -10,28 +10,15 @@ use sql_builder::prelude::*;
use std::collections::HashMap;
use crate::{
db::UserDB,
error::{Error, JsonResult, Result},
users::Authed,
use windmill_common::{
error::{Error, Result},
utils::Pagination,
};
use axum::{
extract::{Extension, Path, Query},
routing::get,
Json, Router,
};
use serde::{Deserialize, Serialize};
use sql_builder::SqlBuilder;
use sqlx::{FromRow, Postgres, Transaction};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_audit))
.route("/get/:id", get(get_audit))
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug)]
#[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")]
pub enum ActionKind {
@@ -99,14 +86,13 @@ pub struct ListAuditLogQuery {
pub after: Option<chrono::DateTime<chrono::Utc>>,
}
async fn list_audit(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(lq): Query<ListAuditLogQuery>,
) -> JsonResult<Vec<AuditLog>> {
let (per_page, offset) = crate::utils::paginate(pagination);
pub async fn list_audit(
mut tx: Transaction<'_, sqlx::Postgres>,
w_id: String,
pagination: Pagination,
lq: ListAuditLogQuery,
) -> Result<Vec<AuditLog>> {
let (per_page, offset) = windmill_common::utils::paginate(pagination);
let mut sqlb = SqlBuilder::select_from("audit")
.field("*")
@@ -136,25 +122,18 @@ async fn list_audit(
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as::<_, AuditLog>(&sql)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
Ok(rows)
}
async fn get_audit(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(id): Path<i32>,
) -> JsonResult<AuditLog> {
let mut tx = user_db.begin(&authed).await?;
pub async fn get_audit(mut tx: Transaction<'_, sqlx::Postgres>, id: i32) -> Result<AuditLog> {
let audit_o = sqlx::query_as::<_, AuditLog>("SELECT * FROM audit WHERE id = $1")
.bind(id)
.fetch_optional(&mut tx)
.await?;
tx.commit().await?;
let audit = crate::utils::not_found_if_none(audit_o, "AuditLog", &id.to_string())?;
Ok(Json(audit))
let audit = windmill_common::utils::not_found_if_none(audit_o, "AuditLog", &id.to_string())?;
Ok(audit)
}
backend/windmill-common/Cargo.toml
+46
View File
@@ -0,0 +1,46 @@
[package]
name = "windmill-common"
version.workspace = true
authors.workspace = true
edition.workspace = true
[features]
default = []
sqlx = ["dep:sqlx"]
hyper = ["dep:hyper"]
tokio = ["dep:tokio"]
axum = ["dep:axum", "dep:tracing"]
reqwest = ["dep:reqwest"]
prometheus = ["dep:tiny_http", "dep:prometheus"]
tracing_init = [
"dep:console-subscriber",
"dep:tracing",
"dep:tracing-subscriber",
]
[lib]
name = "windmill_common"
path = "src/lib.rs"
[dependencies]
hmac.workspace = true
sha2.workspace = true
thiserror.workspace = true
anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
chrono.workspace = true
hex.workspace = true
rand.workspace = true
sqlx = { workspace = true, optional = true, features = ["postgres"] }
uuid.workspace = true
tiny_http = { workspace = true, optional = true }
prometheus = { workspace = true, optional = true }
tracing = { workspace = true, optional = true }
axum = { workspace = true, optional = true }
hyper = { workspace = true, optional = true }
tokio = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
console-subscriber = { workspace = true, optional = true }
tracing-subscriber = { workspace = true, optional = true }
backend/src/error.rs → backend/windmill-common/src/error.rs
+15 -7
View File
@@ -6,17 +6,21 @@
* LICENSE-AGPL for a copy of the license.
*/
#[cfg(feature = "axum")]
use axum::{
body::{self, BoxBody},
response::IntoResponse,
Json,
};
use hyper::{Response, StatusCode};
#[cfg(feature = "sqlx")]
use sqlx::migrate::MigrateError;
use thiserror::Error;
#[cfg(feature = "tokio")]
use tokio::io;
pub type Result<T> = std::result::Result<T, Error>;
#[cfg(feature = "axum")]
pub type JsonResult<T> = std::result::Result<Json<T>, Error>;
#[derive(Debug, Error)]
@@ -34,8 +38,10 @@ pub enum Error {
#[error("{0}")]
ExecutionErr(String),
#[error("IO error: {0}")]
#[cfg(feature = "tokio")]
IoErr(#[from] io::Error),
#[error("Sql error: {0}")]
#[cfg(feature = "sqlx")]
SqlErr(#[from] sqlx::Error),
#[error("Bad request: {0}")]
BadRequest(String),
@@ -44,6 +50,7 @@ pub enum Error {
#[error("Hexadecimal decoding error: {0}")]
HexErr(#[from] hex::FromHexError),
#[error("Migrating database: {0}")]
#[cfg(feature = "sqlx")]
DatabaseMigration(#[from] MigrateError),
#[error("Non-zero exit status: {0}")]
ExitStatus(i32),
@@ -62,18 +69,19 @@ pub fn to_anyhow<T: 'static + std::error::Error + Send + Sync>(e: T) -> anyhow::
From::from(e)
}
#[cfg(feature = "axum")]
impl IntoResponse for Error {
fn into_response(self) -> Response<BoxBody> {
fn into_response(self) -> axum::response::Response<BoxBody> {
let e = &self;
let body = body::boxed(body::Full::from(e.to_string()));
let status = match self {
Self::NotFound(_) => StatusCode::NOT_FOUND,
Self::NotAuthorized(_) => StatusCode::UNAUTHORIZED,
Self::SqlErr(_) | Self::BadRequest(_) => StatusCode::BAD_REQUEST,
_ => StatusCode::INTERNAL_SERVER_ERROR,
Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND,
Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED,
Self::SqlErr(_) | Self::BadRequest(_) => axum::http::StatusCode::BAD_REQUEST,
_ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
};
tracing::error!(error = e.to_string());
Response::builder()
axum::response::Response::builder()
.header("Content-Type", "text/plain")
.status(status)
.body(body)
backend/src/external_ip.rs → backend/windmill-common/src/external_ip.rs
+8
View File
@@ -1,3 +1,11 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Used to determine the internet address that connections from workers will appear to come from.
//!
//! For users writing scripts to access their infrastructure with firewalls requiring incoming
backend/windmill-common/src/flows.rs
+220
View File
@@ -0,0 +1,220 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::{collections::HashMap, time::Duration};
use serde::{self, Deserialize, Serialize};
use crate::{
more_serde::{default_id, default_true, is_default},
scripts::{Schema, ScriptLang},
};
#[derive(Serialize)]
#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
pub struct Flow {
pub workspace_id: String,
pub path: String,
pub summary: String,
pub description: String,
pub value: serde_json::Value,
pub edited_by: String,
pub edited_at: chrono::DateTime<chrono::Utc>,
pub archived: bool,
pub schema: Option<Schema>,
pub extra_perms: serde_json::Value,
}
#[derive(Deserialize)]
#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
pub struct NewFlow {
pub path: String,
pub summary: String,
pub description: String,
pub value: serde_json::Value,
pub schema: Option<Schema>,
}
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct FlowValue {
pub modules: Vec<FlowModule>,
#[serde(default)]
pub failure_module: Option<FlowModule>,
#[serde(default)]
#[serde(skip_serializing_if = "is_default")]
pub same_worker: bool,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct StopAfterIf {
pub expr: String,
pub skip_if_stopped: bool,
}
#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct Retry {
pub constant: ConstantDelay,
pub exponential: ExponentialDelay,
}
impl Retry {
/// Takes the number of previous retries and returns the interval until the next retry if any.
///
/// May return [`Duration::ZERO`] to retry immediately.
pub fn interval(&self, previous_attempts: u16) -> Option<Duration> {
let Self { constant, exponential } = self;
if previous_attempts < constant.attempts {
Some(Duration::from_secs(constant.seconds as u64))
} else if previous_attempts - constant.attempts < exponential.attempts {
let exp = previous_attempts.saturating_add(1) as u32;
let secs = exponential.multiplier * exponential.seconds.saturating_pow(exp);
Some(Duration::from_secs(secs as u64))
} else {
None
}
}
pub fn has_attempts(&self) -> bool {
self.constant.attempts != 0 || self.exponential.attempts != 0
}
pub fn max_attempts(&self) -> u16 {
self.constant
.attempts
.saturating_add(self.exponential.attempts)
}
pub fn max_interval(&self) -> Option<Duration> {
self.max_attempts()
.checked_sub(1)
.and_then(|p| self.interval(p))
}
}
#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct ConstantDelay {
pub attempts: u16,
pub seconds: u16,
}
/// multiplier * seconds ^ failures
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(default)]
pub struct ExponentialDelay {
pub attempts: u16,
pub multiplier: u16,
pub seconds: u16,
}
impl Default for ExponentialDelay {
fn default() -> Self {
Self { attempts: 0, multiplier: 1, seconds: 0 }
}
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct Suspend {
#[serde(skip_serializing_if = "Option::is_none")]
pub required_events: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub timeout: Option<u32>,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct FlowModule {
#[serde(default = "default_id")]
pub id: String,
#[serde(default)]
#[serde(alias = "input_transform")]
pub input_transforms: HashMap<String, InputTransform>,
pub value: FlowModuleValue,
pub stop_after_if: Option<StopAfterIf>,
pub summary: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub suspend: Option<Suspend>,
#[serde(skip_serializing_if = "Option::is_none")]
pub retry: Option<Retry>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sleep: Option<InputTransform>,
}
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum InputTransform {
Static { value: serde_json::Value },
Javascript { expr: String },
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchOneModules {
#[serde(skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
pub expr: String,
pub modules: Vec<FlowModule>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchAllModules {
#[serde(skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
pub modules: Vec<FlowModule>,
#[serde(default = "default_true")]
pub skip_failure: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum FlowModuleValue {
Script {
#[serde(default)]
#[serde(alias = "input_transform")]
input_transforms: HashMap<String, InputTransform>,
path: String,
},
ForloopFlow {
iterator: InputTransform,
modules: Vec<FlowModule>,
#[serde(default = "default_true")]
skip_failures: bool,
},
BranchOne {
branches: Vec<BranchOneModules>,
default: Vec<FlowModule>,
},
BranchAll {
branches: Vec<BranchAllModules>,
},
RawScript {
#[serde(default)]
#[serde(alias = "input_transform")]
input_transforms: HashMap<String, InputTransform>,
content: String,
path: Option<String>,
language: ScriptLang,
},
Identity,
}
#[derive(Deserialize)]
pub struct ListFlowQuery {
pub path_start: Option<String>,
pub path_exact: Option<String>,
pub edited_by: Option<String>,
pub show_archived: Option<bool>,
pub order_by: Option<String>,
pub order_desc: Option<bool>,
}
backend/windmill-common/src/lib.rs
+132
View File
@@ -0,0 +1,132 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::net::SocketAddr;
pub mod error;
pub mod external_ip;
pub mod flows;
pub mod more_serde;
pub mod oauth2;
pub mod scripts;
pub mod users;
pub mod utils;
pub mod variables;
pub mod worker_flow;
#[cfg(feature = "tracing_init")]
pub mod tracing_init;
pub const DEFAULT_NUM_WORKERS: usize = 3;
pub const DEFAULT_TIMEOUT: i32 = 300;
pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
pub const DEFAULT_MAX_CONNECTIONS: u32 = 100;
#[cfg(feature = "tokio")]
pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
use std::io;
use tokio::signal::unix::SignalKind;
async fn terminate() -> io::Result<()> {
tokio::signal::unix::signal(SignalKind::terminate())?
.recv()
.await;
Ok(())
}
tokio::select! {
_ = terminate() => {},
_ = tokio::signal::ctrl_c() => {},
}
println!("signal received, starting graceful shutdown");
let _ = tx.send(());
Ok(())
}
#[cfg(feature = "prometheus")]
pub async fn serve_metrics(
addr: SocketAddr,
rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<(), anyhow::Error> {
use tokio::task::yield_now;
let server = tiny_http::Server::http(addr).map_err(|e| anyhow::anyhow!(e.to_string()))?;
for request in server.incoming_requests() {
yield_now().await;
if !rx.is_empty() {
break;
}
let response = tiny_http::Response::from_string(metrics().await?);
let _ = request.respond(response);
}
Ok(())
}
#[cfg(feature = "prometheus")]
async fn metrics() -> Result<String, error::Error> {
let metric_families = prometheus::gather();
Ok(prometheus::TextEncoder::new()
.encode_to_string(&metric_families)
.map_err(anyhow::Error::from)?)
}
#[cfg(feature = "sqlx")]
pub async fn connect_db() -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
use anyhow::Context;
use error::Error;
let database_url = std::env::var("DATABASE_URL")
.map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
Err(_) => DEFAULT_MAX_CONNECTIONS,
};
Ok(connect(&database_url, max_connections).await?)
}
#[cfg(feature = "sqlx")]
pub async fn connect(
database_url: &str,
max_connections: u32,
) -> Result<sqlx::Pool<sqlx::Postgres>, error::Error> {
use std::time::Duration;
use crate::error::Error;
sqlx::postgres::PgPoolOptions::new()
.max_connections(max_connections)
.max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
.connect(database_url)
.await
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))
}
// TODO: Move this elsewhere
pub async fn get_latest_hash_for_path<'c>(
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
w_id: &str,
script_path: &str,
) -> error::Result<scripts::ScriptHash> {
let script_hash_o = sqlx::query_scalar!(
"select hash from script where path = $1 AND (workspace_id = $2 OR workspace_id = \
'starter') AND
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 OR \
workspace_id = 'starter')) AND
deleted = false",
script_path,
w_id
)
.fetch_optional(db)
.await?;
let script_hash = utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?;
Ok(scripts::ScriptHash(script_hash))
}
backend/src/more_serde.rs → backend/windmill-common/src/more_serde.rs
+8
View File
@@ -1,3 +1,11 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! helpers for serde + serde derive attributes
use crate::utils::rd_string;
backend/windmill-common/src/oauth2.rs
+12
View File
@@ -0,0 +1,12 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use hmac::Hmac;
use sha2::Sha256;
pub type HmacSha256 = Hmac<Sha256>;
backend/windmill-common/src/scripts.rs
+257
View File
@@ -0,0 +1,257 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::{
fmt::Display,
hash::{Hash, Hasher},
};
use serde::de::Error as _;
use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize};
use serde_json::to_string_pretty;
use crate::utils::StripPath;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
#[cfg_attr(
feature = "sqlx",
sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")
)]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum ScriptLang {
Deno,
Python3,
Go,
}
impl ScriptLang {
pub fn as_str(&self) -> &'static str {
match self {
ScriptLang::Deno => "deno",
ScriptLang::Python3 => "python3",
ScriptLang::Go => "go",
}
}
}
#[derive(PartialEq, Debug, Hash, Clone, Copy)]
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
#[cfg_attr(feature = "sqlx", sqlx(transparent))]
pub struct ScriptHash(pub i64);
#[derive(PartialEq)]
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
#[cfg_attr(feature = "sqlx", sqlx(transparent))]
pub struct ScriptHashes(pub Vec<i64>);
impl Display for ScriptHash {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", to_hex_string(&self.0))
}
}
impl Serialize for ScriptHash {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(to_hex_string(&self.0).as_str())
}
}
impl<'de> Deserialize<'de> for ScriptHash {
fn deserialize<D>(deserializer: D) -> std::result::Result<ScriptHash, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
let i = to_i64(&s).map_err(|e| D::Error::custom(format!("{}", e)))?;
Ok(ScriptHash(i))
}
}
impl Serialize for ScriptHashes {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for element in &self.0 {
seq.serialize_element(&ScriptHash(*element))?;
}
seq.end()
}
}
#[derive(Serialize, Deserialize, Debug, Hash)]
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
#[cfg_attr(
feature = "sqlx",
sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")
)]
#[serde(rename_all = "lowercase")]
pub enum ScriptKind {
Trigger,
Failure,
Script,
Approval,
}
#[derive(Serialize)]
#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
pub struct Script {
pub workspace_id: String,
pub hash: ScriptHash,
pub path: String,
pub parent_hashes: Option<ScriptHashes>,
pub summary: String,
pub description: String,
pub content: String,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub archived: bool,
pub schema: Option<Schema>,
pub deleted: bool,
pub is_template: bool,
pub extra_perms: serde_json::Value,
pub lock: Option<String>,
pub lock_error_logs: Option<String>,
pub language: ScriptLang,
pub kind: ScriptKind,
}
#[derive(Serialize, Deserialize, Debug)]
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
#[cfg_attr(feature = "sqlx", sqlx)]
#[cfg_attr(feature = "sqlx", sqlx(transparent))]
#[serde(transparent)]
pub struct Schema(pub serde_json::Value);
impl Hash for Schema {
fn hash<H: Hasher>(&self, state: &mut H) {
if let Ok(s) = to_string_pretty(&self.0) {
s.hash(state);
}
}
}
#[derive(Serialize, Deserialize, Hash)]
pub struct NewScript {
pub path: String,
pub parent_hash: Option<ScriptHash>,
pub summary: String,
pub description: String,
pub content: String,
pub schema: Option<Schema>,
pub is_template: Option<bool>,
pub lock: Option<Vec<String>>,
pub language: ScriptLang,
pub kind: Option<ScriptKind>,
}
#[derive(Deserialize)]
pub struct ListScriptQuery {
pub path_start: Option<String>,
pub path_exact: Option<String>,
pub created_by: Option<String>,
pub first_parent_hash: Option<ScriptHash>,
pub last_parent_hash: Option<ScriptHash>,
pub parent_hash: Option<ScriptHash>,
pub show_archived: Option<bool>,
pub order_by: Option<String>,
pub order_desc: Option<bool>,
pub is_template: Option<bool>,
pub kind: Option<String>,
}
pub fn to_i64(s: &str) -> crate::error::Result<i64> {
let v = hex::decode(s)?;
let nb: u64 = u64::from_be_bytes(
v[0..8]
.try_into()
.map_err(|_| hex::FromHexError::InvalidStringLength)?,
);
Ok(nb as i64)
}
pub fn to_hex_string(i: &i64) -> String {
hex::encode(i.to_be_bytes())
}
#[cfg(feature = "reqwest")]
pub async fn get_hub_script_by_path(
email: Option<String>,
username: String,
path: StripPath,
http_client: reqwest::Client,
host: String,
) -> crate::error::Result<String> {
use crate::{
error::{to_anyhow, Error},
utils::http_get_from_hub,
};
let path = path
.to_path()
.strip_prefix("hub/")
.ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
let content = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/raw/{path}.ts"),
email,
username,
host,
true,
)
.await?
.text()
.await
.map_err(to_anyhow)?;
Ok(content)
}
#[cfg(feature = "reqwest")]
pub async fn get_full_hub_script_by_path(
email: Option<String>,
username: String,
path: StripPath,
http_client: reqwest::Client,
host: String,
) -> crate::error::Result<HubScript> {
use crate::{
error::{to_anyhow, Error},
utils::http_get_from_hub,
};
let path = path
.to_path()
.strip_prefix("hub/")
.ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
let value = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/raw2/{path}"),
email,
username,
host,
true,
)
.await?
.json::<HubScript>()
.await
.map_err(to_anyhow)?;
Ok(value)
}
#[derive(Deserialize, Serialize)]
pub struct HubScript {
pub content: String,
pub lockfile: Option<String>,
pub language: ScriptLang,
pub schema: Option<String>,
}
backend/src/tracing_init.rs → backend/windmill-common/src/tracing_init.rs
+10 -38
View File
@@ -1,46 +1,18 @@
use ::tracing::{field, Metadata, Span};
use ::tracing_subscriber::{
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use tracing::Metadata;
use tracing_subscriber::{
filter::filter_fn,
fmt::{format, Layer},
prelude::*,
EnvFilter,
};
use hyper::Response;
use tower_http::trace::{MakeSpan, OnResponse};
#[derive(Clone)]
pub struct MyOnResponse {}
impl<B> OnResponse<B> for MyOnResponse {
fn on_response(
self,
response: &Response<B>,
latency: std::time::Duration,
_span: &tracing::Span,
) {
tracing::info!(
latency = latency.as_millis(),
status = response.status().as_u16(),
"response"
)
}
}
#[derive(Clone)]
pub struct MyMakeSpan {}
impl<B> MakeSpan<B> for MyMakeSpan {
fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
tracing::info_span!(
"request",
method = %request.method(),
uri = %request.uri(),
username = field::Empty,
workspace_id = field::Empty,
email = field::Empty,
)
}
}
fn json_layer<S>() -> Layer<S, format::JsonFields, format::Format<format::Json>> {
tracing_subscriber::fmt::layer()
backend/windmill-common/src/users.rs
+12
View File
@@ -0,0 +1,12 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
pub fn owner_to_token_owner(user: &str, is_group: bool) -> String {
let prefix = if is_group { 'g' } else { 'u' };
format!("{}/{}", prefix, user)
}
backend/src/utils.rs → backend/windmill-common/src/utils.rs
+16 -35
View File
@@ -7,11 +7,9 @@
*/
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use reqwest::Response;
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
use crate::error::{to_anyhow, Error, Result};
use crate::error::{Error, Result};
pub const MAX_PER_PAGE: usize = 1000;
pub const DEFAULT_PER_PAGE: usize = 100;
@@ -34,26 +32,6 @@ impl StripPath {
}
}
pub async fn require_super_admin<'c>(
db: &mut Transaction<'c, Postgres>,
email: Option<String>,
) -> Result<()> {
let is_admin = sqlx::query_scalar!(
"SELECT super_admin FROM password WHERE email = $1",
email.as_ref()
)
.fetch_one(db)
.await
.map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
if !is_admin {
Err(Error::NotAuthorized(
"This endpoint require caller to be a super admin".to_owned(),
))
} else {
Ok(())
}
}
pub fn require_admin(is_admin: bool, username: &str) -> Result<()> {
if !is_admin {
Err(Error::NotAuthorized(format!(
@@ -65,14 +43,6 @@ pub fn require_admin(is_admin: bool, username: &str) -> Result<()> {
}
}
pub fn rd_string(len: usize) -> String {
thread_rng()
.sample_iter(&Alphanumeric)
.take(len)
.map(char::from)
.collect()
}
pub fn paginate(pagination: Pagination) -> (usize, usize) {
let per_page = pagination
.per_page
@@ -83,8 +53,9 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
(per_page, offset)
}
#[cfg(feature = "sqlx")]
pub async fn now_from_db<'c>(
db: &mut Transaction<'c, Postgres>,
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
) -> Result<chrono::DateTime<chrono::Utc>> {
Ok(sqlx::query_scalar!("SELECT now()")
.fetch_one(db)
@@ -108,6 +79,7 @@ pub fn get_owner_from_path(path: &str) -> String {
path.split('/').take(2).collect::<Vec<_>>().join("/")
}
#[cfg(feature = "reqwest")]
pub async fn list_elems_from_hub(
http_client: reqwest::Client,
url: &str,
@@ -119,10 +91,11 @@ pub async fn list_elems_from_hub(
.await?
.json::<serde_json::Value>()
.await
.map_err(to_anyhow)?;
.map_err(crate::error::to_anyhow)?;
Ok(rows)
}
#[cfg(feature = "reqwest")]
pub async fn http_get_from_hub(
http_client: reqwest::Client,
url: &str,
@@ -130,7 +103,7 @@ pub async fn http_get_from_hub(
username: String,
host: String,
plain: bool,
) -> Result<Response> {
) -> Result<reqwest::Response> {
let response = http_client
.get(url)
.header(
@@ -146,7 +119,15 @@ pub async fn http_get_from_hub(
.header("X-hostname", host)
.send()
.await
.map_err(to_anyhow)?;
.map_err(crate::error::to_anyhow)?;
Ok(response)
}
pub fn rd_string(len: usize) -> String {
thread_rng()
.sample_iter(&Alphanumeric)
.take(len)
.map(char::from)
.collect()
}
backend/windmill-common/src/variables.rs
+118
View File
@@ -0,0 +1,118 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use serde::{Deserialize, Serialize};
#[derive(Serialize, Clone)]
pub struct ContextualVariable {
pub name: String,
pub value: String,
pub description: String,
}
#[derive(Serialize, Deserialize)]
#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
pub struct ListableVariable {
pub workspace_id: String,
pub path: String,
pub value: Option<String>,
pub is_secret: bool,
pub description: String,
pub extra_perms: serde_json::Value,
pub account: Option<i32>,
pub is_oauth: bool,
pub is_expired: Option<bool>,
}
#[derive(Deserialize)]
pub struct CreateVariable {
pub path: String,
pub value: String,
pub is_secret: bool,
pub description: String,
pub account: Option<i32>,
pub is_oauth: Option<bool>,
}
pub fn get_reserved_variables(
w_id: &str,
token: &str,
email: &str,
username: &str,
job_id: &str,
permissioned_as: &str,
base_url: &str,
path: Option<String>,
flow_id: Option<String>,
flow_path: Option<String>,
schedule_path: Option<String>,
) -> [ContextualVariable; 11] {
[
ContextualVariable {
name: "WM_WORKSPACE".to_string(),
value: w_id.to_string(),
description: "Workspace id of the current script".to_string(),
},
ContextualVariable {
name: "WM_TOKEN".to_string(),
value: token.to_string(),
description: "Token ephemeral to the current script with equal permission to the \
permission of the run (Usable as a bearer token)"
.to_string(),
},
ContextualVariable {
name: "WM_EMAIL".to_string(),
value: email.to_string(),
description: "Email of the user that executed the current script".to_string(),
},
ContextualVariable {
name: "WM_USERNAME".to_string(),
value: username.to_string(),
description: "Username of the user that executed the current script".to_string(),
},
ContextualVariable {
name: "WM_BASE_URL".to_string(),
value: base_url.to_string(),
description: "base url of this instance".to_string(),
},
ContextualVariable {
name: "WM_JOB_ID".to_string(),
value: job_id.to_string(),
description: "Job id of the current script".to_string(),
},
ContextualVariable {
name: "WM_JOB_PATH".to_string(),
value: path.unwrap_or_else(|| "".to_string()),
description: "Path of the script or flow being run if any".to_string(),
},
ContextualVariable {
name: "WM_FLOW_JOB_ID".to_string(),
value: flow_id.unwrap_or_else(|| "".to_string()),
description: "Job id of the encapsulating flow if the job is a flow step".to_string(),
},
ContextualVariable {
name: "WM_FLOW_PATH".to_string(),
value: flow_path.unwrap_or_else(|| "".to_string()),
description: "Path of the encapsulating flow if the job is a flow step".to_string(),
},
ContextualVariable {
name: "WM_SCHEDULE_PATH".to_string(),
value: schedule_path.unwrap_or_else(|| "".to_string()),
description: "Path of the schedule if the job of the step or encapsulating step has \
been triggered by a schedule"
.to_string(),
},
ContextualVariable {
name: "WM_PERMISSIONED_AS".to_string(),
value: permissioned_as.to_string(),
description: "Fully Qualified (u/g) owner name of executor of the job".to_string(),
},
]
}
backend/windmill-common/src/worker_flow.rs
+162
View File
@@ -0,0 +1,162 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::time::Duration;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::{flows::FlowValue, more_serde::is_default};
const MINUTES: Duration = Duration::from_secs(60);
const HOURS: Duration = MINUTES.saturating_mul(60);
pub const MAX_RETRY_ATTEMPTS: u16 = 1000;
pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6);
#[derive(Serialize, Deserialize, Debug)]
pub struct FlowStatus {
pub step: i32,
pub modules: Vec<FlowStatusModule>,
pub failure_module: FlowStatusModule,
#[serde(default)]
#[serde(skip_serializing_if = "is_default")]
pub retry: RetryStatus,
}
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct RetryStatus {
pub fail_count: u16,
pub previous_result: Option<serde_json::Value>,
pub failed_jobs: Vec<Uuid>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Iterator {
pub index: usize,
pub itered: Vec<serde_json::Value>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchAllStatus {
pub branch: usize,
pub previous_result: serde_json::Value,
pub len: usize,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum BranchChosen {
Default,
Branch { branch: usize },
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Approval {
pub resume_id: u16,
pub approver: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(tag = "type")]
pub enum FlowStatusModule {
WaitingForPriorSteps {
id: String,
},
WaitingForEvents {
id: String,
count: u16,
job: Uuid,
},
WaitingForExecutor {
id: String,
job: Uuid,
},
InProgress {
id: String,
job: Uuid,
#[serde(skip_serializing_if = "Option::is_none")]
iterator: Option<Iterator>,
#[serde(skip_serializing_if = "Option::is_none")]
flow_jobs: Option<Vec<Uuid>>,
#[serde(skip_serializing_if = "Option::is_none")]
branch_chosen: Option<BranchChosen>,
#[serde(skip_serializing_if = "Option::is_none")]
branchall: Option<BranchAllStatus>,
},
Success {
id: String,
job: Uuid,
#[serde(skip_serializing_if = "Option::is_none")]
flow_jobs: Option<Vec<Uuid>>,
#[serde(skip_serializing_if = "Option::is_none")]
branch_chosen: Option<BranchChosen>,
approvers: Vec<Approval>,
},
Failure {
id: String,
job: Uuid,
#[serde(skip_serializing_if = "Option::is_none")]
flow_jobs: Option<Vec<Uuid>>,
#[serde(skip_serializing_if = "Option::is_none")]
branch_chosen: Option<BranchChosen>,
},
}
impl FlowStatusModule {
pub fn job(&self) -> Option<Uuid> {
match self {
FlowStatusModule::WaitingForPriorSteps { .. } => None,
FlowStatusModule::WaitingForEvents { job, .. } => Some(*job),
FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job),
FlowStatusModule::InProgress { job, .. } => Some(*job),
FlowStatusModule::Success { job, .. } => Some(*job),
FlowStatusModule::Failure { job, .. } => Some(*job),
}
}
pub fn id(&self) -> String {
match self {
FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(),
FlowStatusModule::WaitingForEvents { id, .. } => id.clone(),
FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(),
FlowStatusModule::InProgress { id, .. } => id.clone(),
FlowStatusModule::Success { id, .. } => id.clone(),
FlowStatusModule::Failure { id, .. } => id.clone(),
}
}
}
impl FlowStatus {
pub fn new(f: &FlowValue) -> Self {
Self {
step: 0,
modules: f
.modules
.iter()
.map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() })
.collect(),
failure_module: FlowStatusModule::WaitingForPriorSteps { id: "failure".to_string() },
retry: RetryStatus { fail_count: 0, previous_result: None, failed_jobs: vec![] },
}
}
/// current module status ... excluding failure_module
pub fn current_step(&self) -> Option<&FlowStatusModule> {
let i = usize::try_from(self.step).ok()?;
self.modules.get(i)
}
}
pub fn init_flow_status(f: &FlowValue) -> FlowStatus {
FlowStatus::new(f)
}
backend/windmill-queue/Cargo.toml
+28
View File
@@ -0,0 +1,28 @@
[package]
name = "windmill-queue"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_queue"
path = "src/lib.rs"
[dependencies]
windmill-audit.workspace = true
windmill-common = { workspace = true, features = ["sqlx", "reqwest"] }
anyhow.workspace = true
hmac.workspace = true
sql-builder.workspace = true
sqlx.workspace = true
tracing.workspace = true
serde.workspace = true
serde_json.workspace = true
ulid.workspace = true
uuid.workspace = true
chrono.workspace = true
hex.workspace = true
reqwest.workspace = true
lazy_static.workspace = true
prometheus.workspace = true
cron.workspace = true
backend/windmill-queue/src/jobs.rs
+572
View File
@@ -0,0 +1,572 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::{collections::HashMap, str::FromStr};
use serde::{Deserialize, Serialize};
use sqlx::{Pool, Postgres, Transaction};
use tracing::instrument;
use ulid::Ulid;
use uuid::Uuid;
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{self, to_anyhow, Error},
flows::FlowValue,
scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
utils::StripPath,
worker_flow::{init_flow_status, FlowStatus, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
};
lazy_static::lazy_static! {
// TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens.
static ref QUEUE_PUSH_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
"queue_push_count",
"Total number of jobs pushed to the queue."
)
.unwrap();
static ref QUEUE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
"queue_delete_count",
"Total number of jobs deleted from the queue."
)
.unwrap();
static ref QUEUE_PULL_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
"queue_pull_count",
"Total number of jobs pulled from the queue."
)
.unwrap();
}
const MAX_NB_OF_JOBS_IN_Q_PER_USER: i64 = 10;
const MAX_DURATION_LAST_1200: std::time::Duration = std::time::Duration::from_secs(900);
pub async fn cancel_job<'c>(
username: &str,
reason: Option<String>,
id: Uuid,
w_id: &str,
mut tx: Transaction<'c, Postgres>,
) -> error::Result<(Transaction<'c, Postgres>, Option<Uuid>)> {
let job_option = sqlx::query_scalar!(
"UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 \
AND workspace_id = $4 RETURNING id",
username,
reason,
id,
w_id
)
.fetch_optional(&mut tx)
.await?;
let mut jobs = job_option.map(|j| vec![j]).unwrap_or_default();
while !jobs.is_empty() {
let p_job = jobs.pop();
let new_jobs = sqlx::query_scalar!(
"UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2 WHERE parent_job = $3 \
AND workspace_id = $4 RETURNING id",
username,
reason,
p_job,
w_id
)
.fetch_all(&mut tx)
.await?;
jobs.extend(new_jobs);
}
Ok((tx, job_option))
}
pub async fn pull(db: &Pool<Postgres>) -> windmill_common::error::Result<Option<QueuedJob>> {
/* Jobs can be started if they:
* - haven't been started before,
* running = false
* - are flows with a step that needed resume,
* suspend_until is non-null
* and suspend = 0 when the resume messages are received
* or suspend_until <= now() if it has timed out */
let job: Option<QueuedJob> = sqlx::query_as::<_, QueuedJob>(
"UPDATE queue
SET running = true
, started_at = coalesce(started_at, now())
, last_ping = now()
, suspend_until = null
WHERE id = (
SELECT id
FROM queue
WHERE ( running = false
AND scheduled_for <= now())
OR (suspend_until IS NOT NULL
AND ( suspend <= 0
OR suspend_until <= now()))
ORDER BY scheduled_for
FOR UPDATE SKIP LOCKED
LIMIT 1
)
RETURNING *",
)
.fetch_optional(db)
.await?;
if job.is_some() {
QUEUE_PULL_COUNT.inc();
}
Ok(job)
}
pub async fn get_result_by_id(
db: Pool<Postgres>,
mut skip_direct: bool,
w_id: String,
flow_id: String,
node_id: String,
) -> error::Result<serde_json::Value> {
let mut result_id: Option<Uuid> = None;
let mut parent_id = Uuid::from_str(&flow_id).ok();
while result_id.is_none() && parent_id.is_some() {
if !skip_direct {
let r = sqlx::query!(
"SELECT flow_status, parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT flow_status, parent_job FROM queue WHERE id = $1 AND workspace_id = $2 ",
parent_id.unwrap(),
w_id,
)
.fetch_optional(&db)
.await?;
if let Some(r) = r {
let value = r
.flow_status
.as_ref()
.ok_or_else(|| Error::InternalErr(format!("requiring a flow status value")))?
.to_owned();
parent_id = r.parent_job;
let status_o = serde_json::from_value::<FlowStatus>(value).ok();
result_id = status_o.and_then(|status| {
status
.modules
.iter()
.find(|m| m.id() == node_id)
.and_then(|m| m.job())
});
} else {
parent_id = None;
}
} else {
let q_parent = sqlx::query_scalar!(
"SELECT parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2",
parent_id.unwrap(),
w_id,
)
.fetch_optional(&db)
.await?
.flatten();
parent_id = q_parent;
skip_direct = false
}
}
let result_id = windmill_common::utils::not_found_if_none(
result_id,
"Flow result by id",
format!("{}, {}", flow_id, node_id),
)?;
let value = sqlx::query_scalar!(
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
result_id,
w_id,
)
.fetch_optional(&db)
.await?
.flatten()
.unwrap_or(serde_json::Value::Null);
Ok(value)
}
#[instrument(level = "trace", skip_all)]
pub async fn delete_job(
db: &Pool<Postgres>,
w_id: &str,
job_id: Uuid,
) -> windmill_common::error::Result<()> {
QUEUE_DELETE_COUNT.inc();
let job_removed = sqlx::query_scalar!(
"DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1",
w_id,
job_id
)
.fetch_one(db)
.await
.map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
.unwrap_or(0)
== 1;
tracing::debug!("Job {job_id} deletion was achieved with success: {job_removed}");
Ok(())
}
pub async fn get_queued_job<'c>(
id: Uuid,
w_id: &str,
tx: &mut Transaction<'c, Postgres>,
) -> error::Result<Option<QueuedJob>> {
let r = sqlx::query_as::<_, QueuedJob>(
"SELECT *
FROM queue WHERE id = $1 AND workspace_id = $2",
)
.bind(id)
.bind(w_id)
.fetch_optional(tx)
.await?;
Ok(r)
}
#[instrument(level = "trace", skip_all)]
pub async fn push<'c>(
mut tx: Transaction<'c, Postgres>,
workspace_id: &str,
job_payload: JobPayload,
args: Option<serde_json::Map<String, serde_json::Value>>,
user: &str,
permissioned_as: String,
scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
schedule_path: Option<String>,
parent_job: Option<Uuid>,
is_flow_step: bool,
mut same_worker: bool,
) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now);
let args_json = args.map(serde_json::Value::Object);
let job_id: Uuid = Ulid::new().into();
let premium_workspace =
sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
.fetch_one(&mut tx)
.await
.map_err(|e| {
Error::InternalErr(format!("fetching if {workspace_id} is premium: {e}"))
})?;
if !premium_workspace && std::env::var("CLOUD_HOSTED").is_ok() {
let rate_limiting_queue = sqlx::query_scalar!(
"SELECT COUNT(id) FROM queue WHERE permissioned_as = $1 AND workspace_id = $2",
permissioned_as,
workspace_id
)
.fetch_one(&mut tx)
.await?;
if let Some(nb_jobs) = rate_limiting_queue {
if nb_jobs > MAX_NB_OF_JOBS_IN_Q_PER_USER {
return Err(error::Error::ExecutionErr(format!(
"You have exceeded the number of authorized elements of queue at any given \
time: {}",
MAX_NB_OF_JOBS_IN_Q_PER_USER
)));
}
}
let rate_limiting_duration_ms = sqlx::query_scalar!(
"
SELECT SUM(duration_ms)
FROM completed_job
WHERE permissioned_as = $1
AND created_at > NOW() - INTERVAL '1200 seconds'
AND workspace_id = $2",
permissioned_as,
workspace_id
)
.fetch_one(&mut tx)
.await?;
if let Some(sum_duration_ms) = rate_limiting_duration_ms {
if sum_duration_ms as u128 > MAX_DURATION_LAST_1200.as_millis() {
return Err(error::Error::ExecutionErr(format!(
"You have exceeded the scripts cumulative duration limit over the last 20m \
which is: {} seconds",
MAX_DURATION_LAST_1200.as_secs()
)));
}
}
}
let (script_hash, script_path, raw_code, job_kind, raw_flow, language) = match job_payload {
JobPayload::ScriptHash { hash, path } => {
let language = sqlx::query_scalar!(
"SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND \
(workspace_id = $2 OR workspace_id = 'starter')",
hash.0,
workspace_id
)
.fetch_one(&mut tx)
.await
.map_err(|e| {
Error::InternalErr(format!(
"fetching language for hash {hash} in {workspace_id}: {e}"
))
})?;
(
Some(hash.0),
Some(path),
None,
JobKind::Script,
None,
Some(language),
)
}
JobPayload::ScriptHub { path } => {
let email = sqlx::query_scalar!(
"SELECT email FROM usr WHERE username = $1 AND workspace_id = $2",
user,
workspace_id
)
.fetch_optional(&mut tx)
.await?;
let script = get_hub_script(path.clone(), email, user).await?;
(
None,
Some(path),
Some(script.content.clone()),
JobKind::Script_Hub,
None,
Some(script.language.clone()),
)
}
JobPayload::Code(RawCode { content, path, language }) => (
None,
path,
Some(content),
JobKind::Preview,
None,
Some(language),
),
JobPayload::Dependencies { hash, dependencies, language } => (
Some(hash.0),
None,
Some(dependencies),
JobKind::Dependencies,
None,
Some(language),
),
JobPayload::RawFlow { value, path } => {
(None, path, None, JobKind::FlowPreview, Some(value), None)
}
JobPayload::Flow(flow) => {
let value_json = sqlx::query_scalar!(
"SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = \
'starter')",
flow,
workspace_id
)
.fetch_optional(&mut tx)
.await?
.ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", flow)))?;
let value = serde_json::from_value::<FlowValue>(value_json).map_err(|err| {
Error::InternalErr(format!(
"could not convert json to flow for {flow}: {err:?}"
))
})?;
(None, Some(flow), None, JobKind::Flow, Some(value), None)
}
JobPayload::Identity => (None, None, None, JobKind::Identity, None, None),
};
let is_running = same_worker;
if let Some(flow) = raw_flow.as_ref() {
same_worker = same_worker || flow.same_worker;
for module in flow.modules.iter() {
if let Some(retry) = &module.retry {
if retry.max_attempts() > MAX_RETRY_ATTEMPTS {
Err(Error::BadRequest(format!(
"retry attempts exceeds the maximum of {MAX_RETRY_ATTEMPTS}"
)))?
}
if matches!(retry.max_interval(), Some(interval) if interval > MAX_RETRY_INTERVAL) {
let max = MAX_RETRY_INTERVAL.as_secs();
Err(Error::BadRequest(format!(
"retry interval exceeds the maximum of {max} seconds"
)))?
}
}
}
}
let flow_status = raw_flow.as_ref().map(init_flow_status);
let uuid = sqlx::query_scalar!(
"INSERT INTO queue
(workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for,
script_hash, script_path, raw_code, args, job_kind, schedule_path, raw_flow, \
flow_status, is_flow_step, language, started_at, same_worker)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, CASE WHEN $3 THEN now() END, $18) \
RETURNING id",
workspace_id,
job_id,
is_running,
parent_job,
user,
permissioned_as,
scheduled_for,
script_hash,
script_path.clone(),
raw_code,
args_json,
job_kind: JobKind,
schedule_path,
raw_flow.map(|f| serde_json::json!(f)),
flow_status.map(|f| serde_json::json!(f)),
is_flow_step,
language: ScriptLang,
same_worker
)
.fetch_one(&mut tx)
.await
.map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id}: {e}")))?;
// TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction.
QUEUE_PUSH_COUNT.inc();
{
let uuid_string = job_id.to_string();
let uuid_str = uuid_string.as_str();
let mut hm = HashMap::from([("uuid", uuid_str), ("permissioned_as", &permissioned_as)]);
let s: String;
let operation_name = match job_kind {
JobKind::Preview => "jobs.run.preview",
JobKind::Script => {
s = ScriptHash(script_hash.unwrap()).to_string();
hm.insert("hash", s.as_str());
"jobs.run.script"
}
JobKind::Flow => "jobs.run.flow",
JobKind::FlowPreview => "jobs.run.flow_preview",
JobKind::Script_Hub => "jobs.run.script_hub",
JobKind::Dependencies => "jobs.run.dependencies",
JobKind::Identity => "jobs.run.identity",
};
audit_log(
&mut tx,
&user,
operation_name,
ActionKind::Execute,
workspace_id,
script_path.as_ref().map(|x| x.as_str()),
Some(hm),
)
.await?;
}
Ok((uuid, tx))
}
pub fn canceled_job_to_result(job: &QueuedJob) -> String {
let reason = job
.canceled_reason
.as_deref()
.unwrap_or_else(|| "no reason given");
let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown");
format!("Job canceled: {reason} by {canceler}")
}
pub async fn get_hub_script(
path: String,
email: Option<String>,
user: &str,
) -> error::Result<HubScript> {
get_full_hub_script_by_path(
email,
user.to_string(),
StripPath(path),
reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?,
std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string()),
)
.await
.map(|e| e)
}
#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
pub struct QueuedJob {
pub workspace_id: String,
pub id: Uuid,
pub parent_job: Option<Uuid>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub scheduled_for: chrono::DateTime<chrono::Utc>,
pub running: bool,
pub script_hash: Option<ScriptHash>,
pub script_path: Option<String>,
pub args: Option<serde_json::Value>,
pub logs: Option<String>,
pub raw_code: Option<String>,
pub canceled: bool,
pub canceled_by: Option<String>,
pub canceled_reason: Option<String>,
pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
pub job_kind: JobKind,
pub schedule_path: Option<String>,
pub permissioned_as: String,
pub flow_status: Option<serde_json::Value>,
pub raw_flow: Option<serde_json::Value>,
pub is_flow_step: bool,
pub language: Option<ScriptLang>,
pub same_worker: bool,
}
impl QueuedJob {
pub fn script_path(&self) -> &str {
self.script_path
.as_ref()
.map(String::as_str)
.unwrap_or("NO_FLOW_PATH")
}
}
impl QueuedJob {
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
self.raw_flow
.as_ref()
.and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
}
pub fn parse_flow_status(&self) -> Option<FlowStatus> {
self.flow_status
.as_ref()
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
}
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase"))]
pub enum JobKind {
Script,
#[allow(non_camel_case_types)]
Script_Hub,
Preview,
Dependencies,
Flow,
FlowPreview,
Identity,
}
#[derive(Debug, Clone)]
pub enum JobPayload {
ScriptHub { path: String },
ScriptHash { hash: ScriptHash, path: String },
Code(RawCode),
Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
Flow(String),
RawFlow { value: FlowValue, path: Option<String> },
Identity,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct RawCode {
pub content: String,
pub path: Option<String>,
pub language: ScriptLang,
}
backend/windmill-queue/src/lib.rs
+12
View File
@@ -0,0 +1,12 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
mod jobs;
pub mod schedule;
pub use jobs::*;
backend/src/schedule.rs → backend/windmill-queue/src/schedule.rs
+54 -97
View File
@@ -8,39 +8,16 @@
use std::str::FromStr;
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{self, Error, JsonResult, Result},
jobs::{self, push, JobPayload},
users::Authed,
utils::{get_owner_from_path, now_from_db, Pagination, StripPath},
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use chrono::{DateTime, Duration, FixedOffset};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use sqlx::{query_scalar, FromRow, Postgres, Transaction};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{self, Error, Result},
utils::{get_owner_from_path, not_found_if_none, now_from_db, paginate, Pagination, StripPath},
};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_schedule))
.route("/get/*path", get(get_schedule))
.route("/exists/*path", get(exists_schedule))
.route("/create", post(create_schedule))
.route("/update/*path", post(edit_schedule))
.route("/delete/*path", delete(delete_schedule))
.route("/setenabled/*path", post(set_enabled))
}
pub fn global_service() -> Router {
Router::new().route("/preview", post(preview_schedule))
}
use crate::{push, JobPayload};
#[derive(FromRow, Serialize, Deserialize, Debug)]
pub struct Schedule {
@@ -97,10 +74,10 @@ pub async fn push_scheduled_job<'c>(
return Ok(tx);
}
let mut args: Option<Map<String, Value>> = None;
let mut args: Option<serde_json::Map<String, serde_json::Value>> = None;
if let Some(args_v) = schedule.args {
if let Value::Object(args_m) = args_v {
if let serde_json::Value::Object(args_m) = args_v {
args = Some(args_m)
} else {
return Err(error::Error::ExecutionErr(
@@ -113,7 +90,7 @@ pub async fn push_scheduled_job<'c>(
JobPayload::Flow(schedule.script_path)
} else {
JobPayload::ScriptHash {
hash: jobs::get_latest_hash_for_path(
hash: windmill_common::get_latest_hash_for_path(
&mut tx,
&schedule.workspace_id,
&schedule.script_path,
@@ -140,15 +117,13 @@ pub async fn push_scheduled_job<'c>(
Ok(tx)
}
async fn create_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(ns): Json<NewSchedule>,
pub async fn create_schedule(
mut tx: Transaction<'_, Postgres>,
w_id: String,
ns: NewSchedule,
username: &str,
) -> Result<String> {
cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
let schedule = sqlx::query_as!(
@@ -159,7 +134,7 @@ async fn create_schedule(
ns.path,
ns.schedule,
ns.offset,
&authed.username,
username,
ns.script_path,
ns.is_flow,
ns.args,
@@ -171,7 +146,7 @@ async fn create_schedule(
audit_log(
&mut tx,
&authed.username,
username,
"schedule.create",
ActionKind::Create,
&w_id,
@@ -241,18 +216,17 @@ async fn clear_schedule<'c>(db: &mut Transaction<'c, Postgres>, path: &str) -> R
Ok(())
}
async fn edit_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(es): Json<EditSchedule>,
pub async fn edit_schedule(
mut tx: Transaction<'_, Postgres>,
w_id: String,
path: StripPath,
es: EditSchedule,
username: &String,
) -> Result<String> {
let path = path.to_path();
cron::Schedule::from_str(&es.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
check_flow_conflict(&mut tx, &w_id, &path, es.is_flow, &es.script_path).await?;
clear_schedule(&mut tx, path).await?;
@@ -277,7 +251,7 @@ async fn edit_schedule(
audit_log(
&mut tx,
&authed.username,
username,
"schedule.edit",
ActionKind::Update,
&w_id,
@@ -294,19 +268,15 @@ async fn edit_schedule(
)
.await?;
tx.commit().await?;
Ok(path.to_string())
}
async fn list_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
) -> JsonResult<Vec<Schedule>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let mut tx = user_db.begin(&authed).await?;
pub async fn list_schedule(
mut tx: Transaction<'_, Postgres>,
w_id: String,
pagination: Pagination,
) -> Result<Vec<Schedule>> {
let (per_page, offset) = paginate(pagination);
let rows = sqlx::query_as!(
Schedule,
"SELECT * FROM schedule WHERE workspace_id = $1 ORDER BY edited_at desc LIMIT $2 OFFSET $3",
@@ -317,7 +287,7 @@ async fn list_schedule(
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
Ok(rows)
}
pub async fn get_schedule_opt<'c>(
@@ -335,24 +305,12 @@ pub async fn get_schedule_opt<'c>(
.await?;
Ok(schedule_opt)
}
async fn get_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Schedule> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let schedule_o = get_schedule_opt(&mut tx, &w_id, path).await?;
let schedule = crate::utils::not_found_if_none(schedule_o, "Schedule", path)?;
tx.commit().await?;
Ok(Json(schedule))
}
async fn exists_schedule(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<bool> {
pub async fn exists_schedule(
tx: &mut Transaction<'_, Postgres>,
w_id: String,
path: StripPath,
) -> Result<bool> {
let path = path.to_path();
let exists = sqlx::query_scalar!(
@@ -360,11 +318,11 @@ async fn exists_schedule(
path,
w_id
)
.fetch_one(&db)
.fetch_one(tx)
.await?
.unwrap_or(false);
Ok(Json(exists))
Ok(exists)
}
#[derive(Deserialize)]
@@ -373,9 +331,9 @@ pub struct PreviewPayload {
pub offset: Option<i32>,
}
pub async fn preview_schedule(
Json(PreviewPayload { schedule, offset }): Json<PreviewPayload>,
) -> JsonResult<Vec<DateTime<chrono::Utc>>> {
pub fn preview_schedule(
PreviewPayload { schedule, offset }: PreviewPayload,
) -> Result<Vec<DateTime<chrono::Utc>>> {
let schedule =
cron::Schedule::from_str(&schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let upcoming: Vec<DateTime<chrono::Utc>> = schedule
@@ -383,7 +341,7 @@ pub async fn preview_schedule(
.take(10)
.map(|x| x.into())
.collect();
Ok(Json(upcoming))
Ok(upcoming)
}
fn get_offset(offset: Option<i32>) -> FixedOffset {
@@ -396,14 +354,13 @@ pub struct SetEnabled {
}
pub async fn set_enabled(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(SetEnabled { enabled }): Json<SetEnabled>,
mut tx: Transaction<'_, Postgres>,
w_id: String,
path: StripPath,
SetEnabled { enabled }: SetEnabled,
username: &str,
) -> Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let schedule_o = sqlx::query_as!(
Schedule,
"UPDATE schedule SET enabled = $1 WHERE path = $2 AND workspace_id = $3 RETURNING *",
@@ -414,7 +371,7 @@ pub async fn set_enabled(
.fetch_optional(&mut tx)
.await?;
let schedule = crate::utils::not_found_if_none(schedule_o, "Schedule", path)?;
let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
clear_schedule(&mut tx, path).await?;
@@ -423,7 +380,7 @@ pub async fn set_enabled(
}
audit_log(
&mut tx,
&authed.username,
username,
"schedule.setenabled",
ActionKind::Update,
&w_id,
@@ -438,13 +395,13 @@ pub async fn set_enabled(
))
}
async fn delete_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
pub async fn delete_schedule(
mut tx: Transaction<'_, Postgres>,
w_id: String,
path: StripPath,
username: &str,
) -> Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"DELETE FROM schedule WHERE path = $1 AND workspace_id = $2",
@@ -456,7 +413,7 @@ async fn delete_schedule(
audit_log(
&mut tx,
&authed.username,
username,
"schedule.delete",
ActionKind::Delete,
&w_id,
backend/windmill-worker/Cargo.toml
+42
View File
@@ -0,0 +1,42 @@
[package]
name = "windmill-worker"
version.workspace = true
authors.workspace = true
edition.workspace = true
default-run = "worker"
[[bin]]
name = "worker"
path = "./src/main.rs"
[dependencies]
windmill-queue.workspace = true
windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker.
windmill-common = { workspace = true, features = [
"tokio",
"sqlx",
"prometheus",
"tracing_init",
] }
windmill-api-client.workspace = true
windmill-parser.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-go.workspace = true
windmill-parser-py.workspace = true
sqlx.workspace = true
uuid.workspace = true
tracing.workspace = true
tokio.workspace = true
serde.workspace = true
serde_json.workspace = true
futures.workspace = true
async-recursion.workspace = true
anyhow.workspace = true
itertools.workspace = true
regex.workspace = true
prometheus.workspace = true
lazy_static.workspace = true
chrono.workspace = true
dotenv.workspace = true
rand.workspace = true # TODO: Remove. only used by token creation hack.
deno_core.workspace = true
backend/windmill-worker/README.md
+5
View File
@@ -0,0 +1,5 @@
# Windmill Worker
The worker. Used to process and execute flows & jobs.
This crate exposes both a library as well as a binary target.
nsjail/download.py.config.proto → backend/windmill-worker/nsjail/download.py.config.proto
View File
nsjail/download_deps.py.sh → backend/windmill-worker/nsjail/download_deps.py.sh
View File
nsjail/run.deno.config.proto → backend/windmill-worker/nsjail/run.deno.config.proto
View File
nsjail/run.go.config.proto → backend/windmill-worker/nsjail/run.go.config.proto
View File
nsjail/run.python3.config.proto → backend/windmill-worker/nsjail/run.python3.config.proto
View File
backend/windmill-worker/src/jobs.rs
+175
View File
@@ -0,0 +1,175 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use sqlx::{Pool, Postgres, Transaction};
use tracing::instrument;
use uuid::Uuid;
use windmill_common::error::Error;
use windmill_queue::{delete_job, JobKind, QueuedJob};
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job_error<E: ToString + std::fmt::Debug>(
db: &Pool<Postgres>,
client: &windmill_api_client::Client,
queued_job: &QueuedJob,
logs: String,
e: E,
metrics: Option<crate::worker::Metrics>,
) -> Result<(Uuid, serde_json::Map<String, serde_json::Value>), Error> {
metrics.map(|m| m.worker_execution_failed.inc());
let mut output_map = serde_json::Map::new();
output_map.insert(
"error".to_string(),
serde_json::Value::String(e.to_string()),
);
let a = add_completed_job(
db,
client,
&queued_job,
false,
false,
serde_json::Value::Object(output_map.clone()),
logs,
)
.await?;
Ok((a, output_map))
}
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job(
db: &Pool<Postgres>,
client: &windmill_api_client::Client,
queued_job: &QueuedJob,
success: bool,
skipped: bool,
result: serde_json::Value,
logs: String,
) -> Result<Uuid, Error> {
let mut tx = db.begin().await?;
let job_id = queued_job.id.clone();
sqlx::query!(
"INSERT INTO completed_job AS cj
( workspace_id
, id
, parent_job
, created_by
, created_at
, started_at
, duration_ms
, success
, script_hash
, script_path
, args
, result
, logs
, raw_code
, canceled
, canceled_by
, canceled_reason
, job_kind
, schedule_path
, permissioned_as
, flow_status
, raw_flow
, is_flow_step
, is_skipped
, language )
VALUES ($1, $2, $3, $4, $5, $6, EXTRACT(milliseconds FROM (now() - $6)), $7, $8, $9,\
$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)
ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
queued_job.workspace_id,
queued_job.id,
queued_job.parent_job,
queued_job.created_by,
queued_job.created_at,
queued_job.started_at,
success,
queued_job.script_hash.map(|x| x.0),
queued_job.script_path,
queued_job.args,
result,
logs,
queued_job.raw_code,
queued_job.canceled,
queued_job.canceled_by,
queued_job.canceled_reason,
queued_job.job_kind: JobKind,
queued_job.schedule_path,
queued_job.permissioned_as,
queued_job.flow_status,
queued_job.raw_flow,
queued_job.is_flow_step,
skipped,
queued_job.language: ScriptLang,
)
.execute(&mut tx)
.await
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
let _ = delete_job(db, &queued_job.workspace_id, job_id).await?;
if !queued_job.is_flow_step
&& queued_job.job_kind != JobKind::Flow
&& queued_job.job_kind != JobKind::FlowPreview
&& queued_job.schedule_path.is_some()
&& queued_job.script_path.is_some()
{
tx = schedule_again_if_scheduled(
tx,
client,
queued_job.schedule_path.as_ref().unwrap(),
queued_job.script_path.as_ref().unwrap(),
&queued_job.workspace_id,
)
.await?;
}
tx.commit().await?;
tracing::debug!("Added completed job {}", queued_job.id);
Ok(queued_job.id)
}
#[instrument(level = "trace", skip_all)]
pub async fn schedule_again_if_scheduled<'c>(
mut tx: Transaction<'c, Postgres>,
client: &windmill_api_client::Client,
schedule_path: &str,
script_path: &str,
w_id: &str,
) -> windmill_common::error::Result<Transaction<'c, Postgres>> {
let schedule = client
.get_schedule(w_id, schedule_path)
.await
.map_err(|_| {
Error::InternalErr(format!(
"Could not find schedule {:?} for workspace {}",
schedule_path, w_id
))
})?
.into_inner();
if schedule.enabled && script_path == schedule.script_path {
tx = windmill_queue::schedule::push_scheduled_job(
tx,
windmill_queue::schedule::Schedule {
workspace_id: w_id.to_owned(),
path: schedule.path,
edited_by: schedule.edited_by,
edited_at: schedule.edited_at,
schedule: schedule.schedule,
offset_: schedule.offset as _,
enabled: schedule.enabled,
script_path: schedule.script_path,
is_flow: schedule.is_flow,
args: schedule
.args
.and_then(|e| serde_json::to_value(e).map_or(None, |v| Some(v))),
extra_perms: serde_json::to_value(schedule.extra_perms).expect("hashmap -> json"),
},
)
.await?;
}
Ok(tx)
}
backend/src/js_eval.rs → backend/windmill-worker/src/js_eval.rs
+22 -48
View File
@@ -14,8 +14,7 @@ use regex::Regex;
use serde_json::Value;
use tokio::{sync::oneshot, time::timeout};
use uuid::Uuid;
use crate::{client, error::Error};
use windmill_common::error::Error;
pub struct EvalCreds {
pub workspace: String,
@@ -136,25 +135,6 @@ fn add_closing_bracket(s: &str) -> String {
s
}
pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
let mut context = JsRuntime::new(RuntimeOptions::default());
let code = format!("let x = {}; x", code);
let res = context.execute_script("<anon>", &code);
match res {
Ok(global) => {
let scope = &mut context.handle_scope();
let local = v8::Local::new(scope, global);
let deserialized_value = serde_v8::from_v8::<serde_json::Value>(scope, local);
match deserialized_value {
Ok(value) => Ok(value),
Err(err) => Err(format!("Cannot deserialize value: {:?}", err)),
}
}
Err(err) => Err(format!("Evaling error: {:?}", err)),
}
}
const SPLIT_PAT: &str = ";\n";
async fn eval(
context: &mut JsRuntime,
@@ -279,34 +259,30 @@ async function resource(path) {{
// Ok(path)
// }
// TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client?
#[op]
async fn op_variable(args: Vec<String>) -> Result<String, anyhow::Error> {
let workspace = &args[0];
let path = &args[1];
let token = &args[2];
let base_url = &args[3];
client::get_variable(workspace, path, token, &base_url).await
let client = windmill_api_client::create_client(base_url, token.clone());
let result = client.get_variable(workspace, path, None).await?;
Ok(result.into_inner().value.unwrap_or_else(|| "".to_owned()))
}
#[op]
async fn op_get_result(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
async fn op_get_result(
args: Vec<String>,
) -> Result<windmill_api_client::types::CompletedJob, anyhow::Error> {
let workspace = &args[0];
let id = &args[1];
let token = &args[2];
let base_url = &args[3];
let client = reqwest::Client::new();
let result = client
.get(format!(
"{base_url}/api/w/{workspace}/jobs/completed/get_result/{id}"
))
.bearer_auth(token)
.send()
.await
.map_err(|e| anyhow::anyhow!("error getting result for {id}: {}", e))?
.json::<Option<serde_json::Value>>()
.await
.map_err(|e| anyhow::anyhow!("error getting result for {id}: {}", e))?;
Ok(result)
let client = windmill_api_client::create_client(base_url, token.clone());
let result = client.get_completed_job(workspace, &id.parse()?).await?;
// TODO: verify this works. Previously this returned Option<serde_jons::Value>, now it's statically typed.
Ok(result.into_inner())
}
#[op]
@@ -317,29 +293,27 @@ async fn op_get_id(args: Vec<String>) -> Result<Option<serde_json::Value>, anyho
let base_url = &args[3];
let node_id = &args[4];
let client = reqwest::Client::new();
let client = windmill_api_client::create_client(base_url, token.clone());
let result = client
.get(format!(
"{base_url}/api/w/{workspace}/jobs/result_by_id/{flow_job_id}/{node_id}?skip_direct=true"
))
.bearer_auth(token)
.send()
.result_by_id(workspace, flow_job_id, node_id, Some(true))
.await
.map_err(|e| anyhow::anyhow!("error getting result for flow {flow_job_id} and node {node_id}: {}", e))?
.json::<Option<serde_json::Value>>()
.await
.map_err(|e| anyhow::anyhow!("error getting result for flow {flow_job_id} and node {node_id}: {}", e))?;
.map_or(None, |e| Some(e.into_inner()));
Ok(result)
}
#[op]
async fn op_resource(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
async fn op_resource(
args: Vec<String>,
) -> Result<windmill_api_client::types::Resource, anyhow::Error> {
let workspace = &args[0];
let path = &args[1];
let token = &args[2];
let base_url = &args[3];
client::get_resource(workspace, path, token, &base_url).await
let client = windmill_api_client::create_client(base_url, token.clone());
let result = client.get_resource(workspace, path).await?;
// TODO: verify this works. Previously this returned Option<serde_jons::Value>, now it's statically typed.
Ok(result.into_inner())
}
#[cfg(test)]
backend/windmill-worker/src/lib.rs
+6
View File
@@ -0,0 +1,6 @@
mod jobs;
mod js_eval;
mod worker;
mod worker_flow;
pub use worker::*;
backend/windmill-worker/src/main.rs
+133
View File
@@ -0,0 +1,133 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::{net::SocketAddr, time::Duration};
use anyhow::Context;
use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
use windmill_common::{
error::{self, Error},
utils::rd_string,
};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// dotenv().ok();
windmill_common::tracing_init::initialize_tracing();
let db = async {
let database_url = std::env::var("DATABASE_URL")
.map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
Err(_) => 10,
};
Ok::<Pool<Postgres>, error::Error>(
PgPoolOptions::new()
.max_connections(max_connections)
.max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
.connect(&database_url)
.await
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
)
}
.await?;
let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
.ok()
.map(|s| {
s.parse::<bool>()
.map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
.or_else(|_| s.parse::<SocketAddr>().map(Some))
})
.transpose()?
.flatten();
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
let shutdown_signal = windmill_common::shutdown_signal(tx);
let base_internal_url =
std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
let timeout = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill_common::DEFAULT_TIMEOUT);
let workers_f = async {
let sleep_queue = std::env::var("SLEEP_QUEUE")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE);
let disable_nuser = std::env::var("DISABLE_NUSER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let disable_nsjail = std::env::var("DISABLE_NSJAIL")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let keep_job_dir = std::env::var("KEEP_JOB_DIR")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
tracing::info!(
"DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
{base_url}, SLEEP_QUEUE: {sleep_queue}, TIMEOUT: \
{timeout}, KEEP_JOB_DIR: {keep_job_dir}"
);
let instance_name = rd_string(5);
let ip = windmill_common::external_ip::get_ip()
.await
.unwrap_or_else(|e| {
tracing::warn!(error = e.to_string(), "failed to get external IP");
"unretrievable IP".to_string()
});
let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
windmill_worker::run_worker(
&db.clone(),
timeout,
&instance_name,
worker_name,
1,
1,
&ip,
sleep_queue,
windmill_worker::WorkerConfig {
disable_nsjail,
disable_nuser,
base_internal_url,
base_url,
keep_job_dir,
},
rx.resubscribe(),
)
.await;
Ok(()) as anyhow::Result<()>
};
let metrics_f = async {
match metrics_addr {
Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
.await
.map_err(anyhow::Error::from),
None => Ok(()),
}
};
futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
Ok(())
}
backend/windmill-worker/src/worker.rs
+2221
View File
File diff suppressed because it is too large Load Diff
backend/src/worker_flow.rs → backend/windmill-worker/src/worker_flow.rs
+202 -303
View File
@@ -1,178 +1,44 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::HashMap;
use std::time::Duration;
use crate::{
db::DB,
error::{self, Error},
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, Suspend},
jobs::{
add_completed_job, add_completed_job_error, canceled_job_to_result, get_queued_job, push,
schedule_again_if_scheduled, script_path_to_payload, JobPayload, QueuedJob, RawCode,
},
js_eval::{eval_timeout, EvalCreds, IdContext},
more_serde::is_default,
users::create_token_for_owner,
worker,
};
use crate::jobs::{add_completed_job, add_completed_job_error, schedule_again_if_scheduled};
use crate::js_eval::{eval_timeout, EvalCreds, IdContext};
use crate::worker;
use anyhow::Context;
use async_recursion::async_recursion;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
use serde_json::{json, Map, Value};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use uuid::Uuid;
const MINUTES: Duration = Duration::from_secs(60);
const HOURS: Duration = MINUTES.saturating_mul(60);
pub const MAX_RETRY_ATTEMPTS: u16 = 1000;
pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6);
#[derive(Serialize, Deserialize, Debug)]
pub struct FlowStatus {
pub step: i32,
pub modules: Vec<FlowStatusModule>,
pub failure_module: FlowStatusModule,
#[serde(default)]
#[serde(skip_serializing_if = "is_default")]
pub retry: RetryStatus,
}
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct RetryStatus {
pub fail_count: u16,
pub previous_result: Option<Value>,
pub failed_jobs: Vec<Uuid>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Iterator {
pub index: usize,
pub itered: Vec<Value>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BranchAllStatus {
pub branch: usize,
pub previous_result: serde_json::Value,
pub len: usize,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum BranchChosen {
Default,
Branch { branch: usize },
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Approval {
pub resume_id: u16,
pub approver: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(tag = "type")]
pub enum FlowStatusModule {
WaitingForPriorSteps {
id: String,
use windmill_common::{
error::{self, to_anyhow, Error},
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, Suspend},
worker_flow::{
Approval, BranchAllStatus, BranchChosen, FlowStatus, FlowStatusModule, RetryStatus,
MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL,
},
WaitingForEvents {
id: String,
count: u16,
job: Uuid,
},
WaitingForExecutor {
id: String,
job: Uuid,
},
InProgress {
id: String,
job: Uuid,
#[serde(skip_serializing_if = "Option::is_none")]
iterator: Option<Iterator>,
#[serde(skip_serializing_if = "Option::is_none")]
flow_jobs: Option<Vec<Uuid>>,
#[serde(skip_serializing_if = "Option::is_none")]
branch_chosen: Option<BranchChosen>,
#[serde(skip_serializing_if = "Option::is_none")]
branchall: Option<BranchAllStatus>,
},
Success {
id: String,
job: Uuid,
#[serde(skip_serializing_if = "Option::is_none")]
flow_jobs: Option<Vec<Uuid>>,
#[serde(skip_serializing_if = "Option::is_none")]
branch_chosen: Option<BranchChosen>,
#[serde(default)]
#[serde(skip_serializing_if = "Vec::is_empty")]
approvers: Vec<Approval>,
},
Failure {
id: String,
job: Uuid,
#[serde(skip_serializing_if = "Option::is_none")]
flow_jobs: Option<Vec<Uuid>>,
#[serde(skip_serializing_if = "Option::is_none")]
branch_chosen: Option<BranchChosen>,
},
}
};
impl FlowStatusModule {
pub fn job(&self) -> Option<Uuid> {
match self {
FlowStatusModule::WaitingForPriorSteps { .. } => None,
FlowStatusModule::WaitingForEvents { job, .. } => Some(*job),
FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job),
FlowStatusModule::InProgress { job, .. } => Some(*job),
FlowStatusModule::Success { job, .. } => Some(*job),
FlowStatusModule::Failure { job, .. } => Some(*job),
}
}
type DB = sqlx::Pool<sqlx::Postgres>;
pub fn id(&self) -> String {
match self {
FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(),
FlowStatusModule::WaitingForEvents { id, .. } => id.clone(),
FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(),
FlowStatusModule::InProgress { id, .. } => id.clone(),
FlowStatusModule::Success { id, .. } => id.clone(),
FlowStatusModule::Failure { id, .. } => id.clone(),
}
}
}
impl FlowStatus {
pub fn new(f: &FlowValue) -> Self {
Self {
step: 0,
modules: f
.modules
.iter()
.map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() })
.collect(),
failure_module: FlowStatusModule::WaitingForPriorSteps { id: "failure".to_string() },
retry: RetryStatus { fail_count: 0, previous_result: None, failed_jobs: vec![] },
}
}
/// current module status ... excluding failure_module
pub fn current_step(&self) -> Option<&FlowStatusModule> {
let i = usize::try_from(self.step).ok()?;
self.modules.get(i)
}
}
use windmill_queue::{
canceled_job_to_result, get_queued_job, push, JobPayload, QueuedJob, RawCode,
};
#[async_recursion]
#[instrument(level = "trace", skip_all)]
pub async fn update_flow_status_after_job_completion(
db: &DB,
client: &windmill_api_client::Client,
job: &QueuedJob,
success: bool,
result: serde_json::Value,
@@ -241,9 +107,10 @@ pub async fn update_flow_status_after_job_completion(
let skip_failure = skip_branch_failure || skip_loop_failures;
let (step_counter, new_status) = match module_status {
FlowStatusModule::InProgress { iterator: Some(Iterator { index, itered, .. }), .. }
if (*index + 1 < itered.len() && (success || skip_loop_failures)) =>
{
FlowStatusModule::InProgress {
iterator: Some(windmill_common::worker_flow::Iterator { index, itered, .. }),
..
} if (*index + 1 < itered.len() && (success || skip_loop_failures)) => {
(old_status.step, module_status.clone())
}
FlowStatusModule::InProgress {
@@ -389,6 +256,7 @@ pub async fn update_flow_status_after_job_completion(
{
tx = schedule_again_if_scheduled(
tx,
client,
flow_job.schedule_path.as_ref().unwrap(),
flow_job.script_path.as_ref().unwrap(),
&w_id,
@@ -411,6 +279,7 @@ pub async fn update_flow_status_after_job_completion(
if flow_job.canceled {
add_completed_job_error(
db,
client,
&flow_job,
logs,
&canceled_job_to_result(&flow_job),
@@ -420,6 +289,7 @@ pub async fn update_flow_status_after_job_completion(
} else {
add_completed_job(
db,
client,
&flow_job,
success,
stop_early && skip_if_stop_early.unwrap_or(false),
@@ -433,6 +303,7 @@ pub async fn update_flow_status_after_job_completion(
match handle_flow(
&flow_job,
db,
client,
result.clone(),
same_worker_tx.clone(),
worker_dir,
@@ -443,6 +314,7 @@ pub async fn update_flow_status_after_job_completion(
Err(err) => {
let _ = add_completed_job_error(
db,
client,
&flow_job,
"Unexpected error during flow chaining:\n".to_string(),
err,
@@ -463,6 +335,7 @@ pub async fn update_flow_status_after_job_completion(
if flow_job.parent_job.is_some() {
return Ok(update_flow_status_after_job_completion(
db,
client,
&flow_job,
success,
result,
@@ -573,10 +446,6 @@ async fn compute_bool_from_expr(
}
}
pub fn init_flow_status(f: &FlowValue) -> FlowStatus {
FlowStatus::new(f)
}
pub async fn update_flow_status_in_progress(
db: &DB,
w_id: &str,
@@ -694,6 +563,7 @@ fn flatten_previous_result(last_result: serde_json::Value) -> serde_json::Value
pub async fn handle_flow(
flow_job: &QueuedJob,
db: &sqlx::Pool<sqlx::Postgres>,
client: &windmill_api_client::Client,
last_result: serde_json::Value,
same_worker_tx: Sender<Uuid>,
worker_dir: &str,
@@ -710,6 +580,7 @@ pub async fn handle_flow(
let fake_job = QueuedJob { parent_job: Some(flow_job.id), ..flow_job.clone() };
update_flow_status_after_job_completion(
db,
client,
&fake_job,
true,
serde_json::json!({}),
@@ -734,6 +605,7 @@ pub async fn handle_flow(
status,
flow,
db,
client,
last_result,
same_worker_tx,
base_internal_url,
@@ -749,10 +621,11 @@ async fn push_next_flow_job(
mut status: FlowStatus,
flow: FlowValue,
db: &sqlx::Pool<sqlx::Postgres>,
client: &windmill_api_client::Client,
mut last_result: serde_json::Value,
same_worker_tx: Sender<Uuid>,
base_internal_url: &str,
) -> anyhow::Result<()> {
) -> error::Result<()> {
/* `mut` because reassigned on FlowStatusModule::Failure when failure_module is Some */
let mut i = usize::try_from(status.step)
.with_context(|| format!("invalid module index {}", status.step))?;
@@ -898,7 +771,7 @@ async fn push_next_flow_job(
)
.bind(json!(FlowStatusModule::WaitingForEvents { id: status_module.id(), count: required_events, job: last }))
.bind((required_events - resume_messages.len() as u16) as i32)
.bind(suspend.timeout.map(|t| Duration::from_secs(t.into())).unwrap_or_else(|| 30 * MINUTES))
.bind(Duration::from_secs(suspend.timeout.map(|t| t.into()).unwrap_or_else(|| 30 * 60)))
.bind(flow_job.id)
.execute(&mut tx)
.await?;
@@ -915,7 +788,9 @@ async fn push_next_flow_job(
let logs = "Timed out waiting to be resumed".to_string();
let result = json!({ "error": logs });
let _uuid =
add_completed_job(db, &flow_job, success, skipped, result, logs).await?;
add_completed_job(db, client, &flow_job, success, skipped, result, logs)
.await?;
return Ok(());
}
}
@@ -1024,12 +899,14 @@ async fn push_next_flow_job(
_ => (),
}
let mut transform_context: Option<(String, Vec<Uuid>, IdContext)> = None;
let mut transform_context: Option<TransformContext> = None;
let mut args = match &module.value {
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. } => {
transform_context =
Some(get_transform_context(&db, &flow_job, &status, &flow.modules).await?);
let tx = db.begin().await?;
let (tx, ctx) = get_transform_context(tx, &flow_job, &status, &flow.modules).await?;
transform_context = Some(ctx);
tx.commit().await?;
let (token, steps, by_id) = transform_context.as_ref().unwrap();
transform_input(
&flow_job.args,
@@ -1071,11 +948,12 @@ async fn push_next_flow_job(
}
};
let next_flow_transform = compute_next_flow_transform(
let tx = db.begin().await?;
let (tx, next_flow_transform) = compute_next_flow_transform(
flow_job,
&flow,
transform_context,
&db,
tx,
&module,
&status,
&status_module,
@@ -1083,6 +961,7 @@ async fn push_next_flow_job(
base_internal_url,
)
.await?;
tx.commit().await?;
let (job_payload, next_status) = match next_flow_transform {
NextFlowTransform::Continue(job_payload, next_state) => (job_payload, next_state),
@@ -1093,6 +972,7 @@ async fn push_next_flow_job(
&flow_job.id,
flow.clone(),
&db,
client,
FlowStatusModule::Success {
id: status_module.id(),
job: flow_job.id,
@@ -1154,7 +1034,7 @@ async fn push_next_flow_job(
FlowStatusModule::InProgress {
job: uuid,
iterator: Some(Iterator { index, itered }),
iterator: Some(windmill_common::worker_flow::Iterator { index, itered }),
flow_jobs: Some(flow_jobs),
branch_chosen: None,
branchall: None,
@@ -1206,7 +1086,7 @@ async fn push_next_flow_job(
tx.commit().await?;
if continue_on_same_worker {
same_worker_tx.send(uuid).await?;
same_worker_tx.send(uuid).await.map_err(to_anyhow)?;
}
return Ok(());
}
@@ -1217,11 +1097,12 @@ async fn jump_to_next_step(
job_id: &Uuid,
flow: FlowValue,
db: &DB,
client: &windmill_api_client::Client,
status_module: FlowStatusModule,
last_result: serde_json::Value,
same_worker_tx: Sender<Uuid>,
base_internal_url: &str,
) -> anyhow::Result<()> {
) -> error::Result<()> {
let mut tx = db.begin().await?;
let next_step = i
@@ -1258,6 +1139,7 @@ async fn jump_to_next_step(
new_status,
flow,
db,
client,
last_result,
same_worker_tx,
base_internal_url,
@@ -1267,7 +1149,8 @@ async fn jump_to_next_step(
let success = true;
let skipped = false;
let logs = "Forloop completed without iteration".to_string();
let _uuid = add_completed_job(db, &new_job, success, skipped, json!([]), logs).await?;
let _uuid =
add_completed_job(db, client, &new_job, success, skipped, json!([]), logs).await?;
return Ok(());
}
}
@@ -1303,38 +1186,62 @@ enum NextFlowTransform {
Continue(JobPayload, NextStatus),
}
async fn compute_next_flow_transform(
// a similar function exists on the backend
// TODO: rewrite this to use an endpoint in the backend directly, instead of checking for hub itself, and then using the API
async fn script_path_to_payload<'c>(
script_path: &str,
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
w_id: &String,
) -> Result<JobPayload, Error> {
let job_payload = if script_path.starts_with("hub/") {
JobPayload::ScriptHub { path: script_path.to_owned() }
} else {
let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
};
Ok(job_payload)
}
type TransformContext = (String, Vec<Uuid>, IdContext);
async fn compute_next_flow_transform<'c>(
flow_job: &QueuedJob,
flow: &FlowValue,
transform_context: Option<(String, Vec<Uuid>, IdContext)>,
db: &DB,
transform_context: Option<TransformContext>,
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
module: &FlowModule,
status: &FlowStatus,
status_module: &FlowStatusModule,
last_result: serde_json::Value,
base_internal_url: &str,
) -> error::Result<NextFlowTransform> {
) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, NextFlowTransform)> {
match &module.value {
FlowModuleValue::Identity => Ok(NextFlowTransform::Continue(
JobPayload::Identity,
NextStatus::NextStep,
)),
FlowModuleValue::Script { path: script_path, .. } => Ok(NextFlowTransform::Continue(
script_path_to_payload(script_path, &mut db.begin().await?, &flow_job.workspace_id)
.await?,
NextStatus::NextStep,
FlowModuleValue::Identity => Ok((
tx,
NextFlowTransform::Continue(JobPayload::Identity, NextStatus::NextStep),
)),
FlowModuleValue::Script { path: script_path, .. } => {
let payload =
script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?;
Ok((
tx,
NextFlowTransform::Continue(payload, NextStatus::NextStep),
))
}
FlowModuleValue::RawScript { path, content, language, .. } => {
let path = path
.clone()
.or_else(|| Some(format!("{}/{}", flow_job.script_path(), status.step)));
Ok(NextFlowTransform::Continue(
JobPayload::Code(RawCode {
path,
content: content.clone(),
language: language.clone(),
}),
NextStatus::NextStep,
Ok((
tx,
NextFlowTransform::Continue(
JobPayload::Code(RawCode {
path,
content: content.clone(),
language: language.clone(),
}),
NextStatus::NextStep,
),
))
}
/* forloop modules are expected set `iter: { value: Value, index: usize }` as job arguments */
@@ -1346,31 +1253,31 @@ async fn compute_next_flow_transform(
let (token, steps, by_id) = if let Some(x) = transform_context {
x
} else {
get_transform_context(&db, &flow_job, &status, &flow.modules).await?
let (tx_new, res) =
get_transform_context(tx, &flow_job, &status, &flow.modules).await?;
tx = tx_new;
res
};
/* Iterator is an InputTransform, evaluate it into an array. */
let itered = iterator
.clone()
.evaluate_with(
|| {
vec![
("result".to_string(), last_result.clone()),
("previous_result".to_string(), last_result.clone()),
]
},
token,
flow_job.workspace_id.clone(),
steps,
Some(by_id),
base_internal_url,
)
.await?
.into_array()
.map_err(|not_array| {
Error::ExecutionErr(format!(
"Expected an array value, found: {not_array}"
))
})?;
let itered = evaluate_with(
iterator.clone(),
|| {
vec![
("result".to_string(), last_result.clone()),
("previous_result".to_string(), last_result.clone()),
]
},
token,
flow_job.workspace_id.clone(),
steps,
Some(by_id),
base_internal_url,
)
.await?
.into_array()
.map_err(|not_array| {
Error::ExecutionErr(format!("Expected an array value, found: {not_array}"))
})?;
if let Some(first) = itered.first() {
new_args.insert("iter".to_string(), json!({ "index": 0, "value": first }));
@@ -1387,7 +1294,7 @@ async fn compute_next_flow_transform(
}
FlowStatusModule::InProgress {
iterator: Some(Iterator { itered, index }),
iterator: Some(windmill_common::worker_flow::Iterator { itered, index }),
flow_jobs: Some(flow_jobs),
..
} => {
@@ -1418,17 +1325,20 @@ async fn compute_next_flow_transform(
};
match next_loop_status {
LoopStatus::EmptyIterator => Ok(NextFlowTransform::EmptyInnerFlows),
LoopStatus::NextIteration(ns) => Ok(NextFlowTransform::Continue(
JobPayload::RawFlow {
value: FlowValue {
modules: (*modules).clone(),
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
LoopStatus::EmptyIterator => Ok((tx, NextFlowTransform::EmptyInnerFlows)),
LoopStatus::NextIteration(ns) => Ok((
tx,
NextFlowTransform::Continue(
JobPayload::RawFlow {
value: FlowValue {
modules: (*modules).clone(),
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
},
path: Some(format!("{}/loop-{}", flow_job.script_path(), status.step)),
},
path: Some(format!("{}/loop-{}", flow_job.script_path(), status.step)),
},
NextStatus::NextLoopIteration(ns),
NextStatus::NextLoopIteration(ns),
),
)),
}
}
@@ -1469,27 +1379,30 @@ async fn compute_next_flow_transform(
default.clone()
};
Ok(NextFlowTransform::Continue(
JobPayload::RawFlow {
value: FlowValue {
modules,
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
Ok((
tx,
NextFlowTransform::Continue(
JobPayload::RawFlow {
value: FlowValue {
modules,
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
},
path: Some(format!(
"{}/branchone-{}",
flow_job.script_path(),
status.step
)),
},
path: Some(format!(
"{}/branchone-{}",
flow_job.script_path(),
status.step
)),
},
NextStatus::BranchChosen(branch),
NextStatus::BranchChosen(branch),
),
))
}
FlowModuleValue::BranchAll { branches, .. } => {
let (status, flow_jobs) = match status_module {
FlowStatusModule::WaitingForPriorSteps { .. } => {
if branches.is_empty() {
return Ok(NextFlowTransform::EmptyInnerFlows);
return Ok((tx, NextFlowTransform::EmptyInnerFlows));
} else {
(
BranchAllStatus {
@@ -1528,33 +1441,36 @@ async fn compute_next_flow_transform(
))
})?;
Ok(NextFlowTransform::Continue(
JobPayload::RawFlow {
value: FlowValue {
modules,
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
Ok((
tx,
NextFlowTransform::Continue(
JobPayload::RawFlow {
value: FlowValue {
modules,
failure_module: flow.failure_module.clone(),
same_worker: flow.same_worker,
},
path: Some(format!(
"{}/branchall-{}",
flow_job.script_path(),
status.branch
)),
},
path: Some(format!(
"{}/branchall-{}",
flow_job.script_path(),
status.branch
)),
},
NextStatus::NextBranchStep(NextBranch { status, flow_jobs }),
NextStatus::NextBranchStep(NextBranch { status, flow_jobs }),
),
))
}
}
}
async fn get_transform_context(
db: &DB,
async fn get_transform_context<'c>(
tx: sqlx::Transaction<'c, sqlx::Postgres>,
flow_job: &QueuedJob,
status: &FlowStatus,
modules: &Vec<FlowModule>,
) -> error::Result<(String, Vec<Uuid>, IdContext)> {
let new_token = create_token_for_owner(
db,
) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, TransformContext)> {
let (tx, new_token) = crate::create_token_for_owner(
tx,
&flow_job.workspace_id,
&flow_job.permissioned_as,
"transform-input",
@@ -1573,53 +1489,36 @@ async fn get_transform_context(
.zip(new_steps.clone())
.collect();
Ok((new_token, new_steps, IdContext(flow_job.id, id_map)))
Ok((tx, (new_token, new_steps, IdContext(flow_job.id, id_map))))
}
impl InputTransform {
async fn evaluate_with<F>(
self,
vars: F,
token: String,
workspace: String,
steps: Vec<Uuid>,
by_id: Option<IdContext>,
base_internal_url: &str,
) -> anyhow::Result<Value>
where
F: FnOnce() -> Vec<(String, Value)>,
{
match self {
InputTransform::Static { value } => Ok(value),
InputTransform::Javascript { expr } => {
eval_timeout(
expr,
vars(),
Some(EvalCreds { workspace, token }),
steps,
by_id,
base_internal_url.to_string(),
)
.await
}
async fn evaluate_with<F>(
transform: InputTransform,
vars: F,
token: String,
workspace: String,
steps: Vec<Uuid>,
by_id: Option<IdContext>,
base_internal_url: &str,
) -> anyhow::Result<serde_json::Value>
where
F: FnOnce() -> Vec<(String, serde_json::Value)>,
{
match transform {
InputTransform::Static { value } => Ok(value),
InputTransform::Javascript { expr } => {
eval_timeout(
expr,
vars(),
Some(EvalCreds { workspace, token }),
steps,
by_id,
base_internal_url.to_string(),
)
.await
}
}
}
impl QueuedJob {
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
self.raw_flow
.as_ref()
.and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
}
pub fn parse_flow_status(&self) -> Option<FlowStatus> {
self.flow_status
.as_ref()
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
}
}
trait IntoArray: Sized {
fn into_array(self) -> Result<Vec<Value>, Self>;
}
deno-client/build.sh
+1 -1
View File
@@ -2,7 +2,7 @@
set -e
npm ci --ignore-scripts
npx --yes openapi-typescript-codegen --input ../backend/openapi.yaml \
npx --yes openapi-typescript-codegen --input ../backend/windmill-api/openapi.yaml \
--output ./src --useOptions \
&& sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/core/request.ts
npx --yes denoify
frontend/package.json
+1 -1
View File
@@ -10,7 +10,7 @@
"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
"format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. .",
"package": "svelte-kit package && cd package && rm README.md && rm README_DEV.md && sed -i -e 's/windmill/windmill-components/g' package.json",
"generate-backend-client": "openapi --input ../backend/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts",
"generate-backend-client": "openapi --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts",
"pretest": "tsc --incremental -p tests/tsconfig.json",
"test": "playwright test --config=tests-out/playwright.config.js"
},
go-client/build.sh
+1 -1
View File
@@ -1,7 +1,7 @@
#!/bin/bash
set -e
cp ../backend/openapi.yaml openapi.yaml
cp ../backend/windmill-api/openapi.yaml openapi.yaml
sed -z 's/ extra_params:\n additionalProperties:\n type: string/ extra_params: {}/' openapi.yaml > openapi1.yaml
sed -z 's/ enum: \[script, failure, trigger, command\]//' openapi1.yaml > openapi2.yaml
openflow.openapi.yaml
+2
View File
@@ -12,6 +12,8 @@ info:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0.html
paths: {}
externalDocs:
description: documentation portal
url: https://docs.windmill.dev
python-client/build.sh
+1 -1
View File
@@ -1,7 +1,7 @@
#!/bin/bash
set -e
cp ../backend/openapi.yaml openapi.yaml
cp ../backend/windmill-api/openapi.yaml openapi.yaml
npx @redocly/openapi-cli@latest bundle openapi.yaml > openapi-bundled.yaml