starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-05 16:03:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

send a webhook on new user + lazy static refactor (#1203)

Browse Source 
* supercharge

* supercharge

* progress

* progress

* display config

* display config

* display config

* display config

* fix extensions

* fix build

* disable nsjail = false for test
This commit is contained in:
Ruben Fiszel
2023-02-15 10:59:50 +01:00
committed by GitHub
parent 81f64a4028
commit c0b87cc7d7
31 changed files with 681 additions and 990 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-test.yml
+1 -1
View File
@@ -40,4 +40,4 @@ jobs:
backend -> target
- name: cargo test
timeout-minutes: 10
run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture
README.md
+3 -3
View File
@@ -302,7 +302,8 @@ you to have it being synced automatically everyday.
| Environment Variable name | Default | Description | Api Server/Worker/All |
| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
| DATABASE_URL | | The Postgres database url. | All |
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | |
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker |
| PORT | 8000 | Exposed port | Server | |
| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker |
| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker |
| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
@@ -318,8 +319,7 @@ you to have it being synced automatically everyday.
| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker |
| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker |
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| SERVE_CSP | None | The CSP directives to use when serving the frontend static assets | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
backend/Cargo.lock
Generated
+1
View File
@@ -4785,6 +4785,7 @@ dependencies = [
"hex",
"hmac",
"hyper",
"lazy_static",
"prometheus",
"rand 0.8.5",
"reqwest",
backend/src/main.rs
+81 -85
View File
@@ -11,9 +11,10 @@ use std::net::SocketAddr;
use git_version::git_version;
use sqlx::{Pool, Postgres};
use windmill_common::utils::rd_string;
use windmill_worker::WorkerConfig;
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
const DEFAULT_NUM_WORKERS: usize = 3;
const DEFAULT_PORT: u16 = 8000;
mod ee;
@@ -26,7 +27,7 @@ async fn main() -> anyhow::Result<()> {
let num_workers = std::env::var("NUM_WORKERS")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32);
.unwrap_or(DEFAULT_NUM_WORKERS as i32);
let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
.ok()
@@ -38,6 +39,13 @@ async fn main() -> anyhow::Result<()> {
.transpose()?
.flatten();
let port: u16 = std::env::var("PORT")
.ok()
.and_then(|x| x.parse::<u16>().ok())
.unwrap_or(DEFAULT_PORT as u16);
let base_internal_url: String = std::env::var("BASE_INTERNAL_URL")
.unwrap_or_else(|_| format!("http://localhost:{}", port.to_string()));
let server_mode = !std::env::var("DISABLE_SERVER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
@@ -52,58 +60,25 @@ async fn main() -> anyhow::Result<()> {
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
let shutdown_signal = windmill_common::shutdown_signal(tx);
let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
let base_internal_url =
std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
let timeout = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill_common::DEFAULT_TIMEOUT);
if server_mode || num_workers > 0 {
let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
let addr = SocketAddr::from(([0, 0, 0, 0], port));
let base_url2 = base_url.clone();
let server_f = async {
if server_mode {
windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?;
windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
}
Ok(()) as anyhow::Result<()>
};
let base_url = base_url2.clone();
let workers_f = async {
if num_workers > 0 {
let sleep_queue = std::env::var("SLEEP_QUEUE")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE);
let disable_nuser = std::env::var("DISABLE_NUSER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let disable_nsjail = std::env::var("DISABLE_NSJAIL")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(true);
let keep_job_dir = std::env::var("KEEP_JOB_DIR")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let license_key = std::env::var("LICENSE_KEY").ok();
let sync_bucket = std::env::var("S3_CACHE_BUCKET")
.ok()
.map(|e| Some(e))
.unwrap_or(None);
#[cfg(feature = "enterprise")]
tracing::info!(
"
"
##############################
Windmill Enterprise Edition {GIT_VERSION} LICENSE_KEY: {license_key:?}, S3_CACHE_BUCKET: {sync_bucket:?}
Windmill Enterprise Edition {GIT_VERSION}
##############################"
);
);
#[cfg(not(feature = "enterprise"))]
tracing::info!(
@@ -113,38 +88,58 @@ Windmill Community Edition {GIT_VERSION}
##############################"
);
tracing::info!(
"DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
{base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
{timeout}, KEEP_JOB_DIR: {keep_job_dir}"
);
display_config(vec![
"DISABLE_NSJAIL",
"DISABLE_SERVER",
"NUM_WORKERS",
"METRICS_ADDR",
"JSON_FMT",
"BASE_URL",
"BASE_INTERNAL_URL",
"TIMEOUT",
"SLEEP_QUEUE",
"MAX_LOG_SIZE",
"PORT",
"KEEP_JOB_DIR",
"S3_CACHE_BUCKET",
"TAR_CACHE_RATE",
"COOKIE_DOMAIN",
"PYTHON_PATH",
"DENO_PATH",
"GO_PATH",
"PIP_INDEX_URL",
"PIP_EXTRA_INDEX_URL",
"PIP_TRUSTED_HOST",
"PATH",
"HOME",
"DATABASE_CONNECTIONS",
"TIMEOUT_WAIT_RESULT",
"QUEUE_LIMIT_WAIT_RESULT",
"DENO_AUTH_TOKENS",
"DENO_FLAGS",
"PIP_LOCAL_DEPENDENCIES",
"ADDITIONAL_PYTHON_PATHS",
"INCLUDE_HEADERS",
"WHITELIST_WORKSPACES",
"BLACKLIST_WORKSPACES",
"NEW_USER_WEBHOOK",
"CLOUD_HOSTED",
]);
run_workers(
db.clone(),
addr,
timeout,
num_workers,
sleep_queue,
WorkerConfig {
disable_nsjail,
disable_nuser,
base_internal_url,
base_url,
keep_job_dir,
},
rx.resubscribe(),
sync_bucket,
license_key,
num_workers,
base_internal_url.clone(),
)
.await?;
}
Ok(()) as anyhow::Result<()>
};
let base_url = base_url2;
let monitor_f = async {
if server_mode {
monitor_db(&db, timeout, base_url, rx.resubscribe());
monitor_db(&db, rx.resubscribe(), &base_internal_url);
}
Ok(()) as anyhow::Result<()>
};
@@ -163,34 +158,46 @@ Windmill Community Edition {GIT_VERSION}
Ok(())
}
fn display_config(envs: Vec<&str>) {
tracing::info!(
"config: {}",
envs.iter()
.filter(|env| std::env::var(env).is_ok())
.map(|env| {
format!(
"{}: {}",
env,
std::env::var(env).unwrap_or_else(|_| "not set".to_string())
)
})
.collect::<Vec<String>>()
.join(", ")
)
}
pub fn monitor_db(
db: &Pool<Postgres>,
timeout: i32,
base_url: String,
rx: tokio::sync::broadcast::Receiver<()>,
base_internal_url: &str,
) {
let db1 = db.clone();
let db2 = db.clone();
let rx2 = rx.resubscribe();
let base_internal_url = base_internal_url.to_string();
tokio::spawn(async move {
windmill_worker::handle_zombie_jobs_periodically(&db1, timeout, &base_url, rx).await
windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url).await
});
tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
}
pub async fn run_workers(
db: Pool<Postgres>,
addr: SocketAddr,
timeout: i32,
num_workers: i32,
sleep_queue: u64,
worker_config: WorkerConfig,
rx: tokio::sync::broadcast::Receiver<()>,
mut periodic_script: Option<String>,
license_key: Option<String>,
num_workers: i32,
base_internal_url: String,
) -> anyhow::Result<()> {
let license_key = std::env::var("LICENSE_KEY").ok();
#[cfg(feature = "enterprise")]
ee::verify_license_key(license_key)?;
@@ -198,12 +205,6 @@ pub async fn run_workers(
if license_key.is_some() {
panic!("License key is required ONLY for the enterprise edition");
}
#[cfg(not(feature = "enterprise"))]
if !worker_config.disable_nsjail {
tracing::warn!(
"NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes"
);
}
let instance_name = rd_string(5);
let monitor = tokio_metrics::TaskMonitor::new();
@@ -223,22 +224,17 @@ pub async fn run_workers(
let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
let ip = ip.clone();
let rx = rx.resubscribe();
let worker_config = worker_config.clone();
let wp = periodic_script.take();
let base_internal_url = base_internal_url.clone();
handles.push(tokio::spawn(monitor.instrument(async move {
tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
tracing::info!(worker = %worker_name, "starting worker");
windmill_worker::run_worker(
&db1,
timeout,
&instance_name,
worker_name,
i as u64,
num_workers as u64,
&ip,
sleep_queue,
worker_config,
wp,
rx,
&base_internal_url,
)
.await
})));
backend/tests/worker.rs
+3 -35
View File
@@ -6,10 +6,8 @@ use windmill_common::{
flow_status::{FlowStatus, FlowStatusModule},
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
scripts::ScriptLang,
DEFAULT_SLEEP_QUEUE,
};
use windmill_queue::{get_queued_job, JobPayload, RawCode};
use windmill_worker::WorkerConfig;
async fn initialize_tracing() {
use std::sync::Once;
@@ -89,14 +87,7 @@ impl ApiServer {
let addr = sock.local_addr().unwrap();
drop(sock);
let task = tokio::task::spawn({
windmill_api::run_server(
db.clone(),
addr,
format!("http://localhost:{}", addr.port()),
rx,
)
});
let task = tokio::task::spawn(windmill_api::run_server(db.clone(), addr, rx));
return Self { addr, tx, task };
}
@@ -917,43 +908,20 @@ fn spawn_test_worker(
) {
let (tx, rx) = tokio::sync::broadcast::channel(1);
let db = db.to_owned();
let timeout = 4_000;
let worker_instance: &str = "test worker instance";
let worker_name: String = next_worker_name();
let i_worker: u64 = Default::default();
let num_workers: u64 = 2;
let ip: &str = Default::default();
let sleep_queue: u64 = DEFAULT_SLEEP_QUEUE / num_workers;
let port = port;
let worker_config = WorkerConfig {
base_internal_url: format!("http://localhost:{port}"),
base_url: format!("http://localhost:{port}"),
disable_nuser: std::env::var("DISABLE_NUSER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false),
disable_nsjail: std::env::var("DISABLE_NSJAIL")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false),
keep_job_dir: std::env::var("KEEP_JOB_DIR")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false),
};
let future = async move {
let base_internal_url = format!("http://localhost:{}", port);
windmill_worker::run_worker(
&db,
timeout,
worker_instance,
worker_name,
i_worker,
num_workers,
ip,
sleep_queue,
worker_config,
None,
rx,
&base_internal_url,
)
.await
};
backend/windmill-api/Cargo.toml
-4
View File
@@ -8,10 +8,6 @@ edition.workspace = true
name = "windmill_api"
path = "src/lib.rs"
[[bin]]
name = "windmill_api"
path = "src/main.rs"
[features]
enterprise = ["windmill-queue/enterprise"]
backend/windmill-api/src/apps.rs
+4 -8
View File
@@ -13,6 +13,7 @@ use crate::{
users::{require_owner_of_path, Authed, OptAuthed},
variables::build_crypt,
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
use axum::{
extract::{Extension, Json, Path, Query},
@@ -21,7 +22,6 @@ use axum::{
};
use hyper::StatusCode;
use magic_crypt::MagicCryptTrait;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::{json, Map, Value};
use sha2::{Digest, Sha256};
@@ -368,12 +368,9 @@ async fn create_app(
Ok((StatusCode::CREATED, app.path))
}
async fn list_hub_apps(
Authed { email, .. }: Authed,
Extension(http_client): Extension<Client>,
) -> JsonResult<serde_json::Value> {
async fn list_hub_apps(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
let flows = list_elems_from_hub(
http_client,
&HTTP_CLIENT,
"https://hub.windmill.dev/searchUiData?approved=true",
&email,
)
@@ -384,10 +381,9 @@ async fn list_hub_apps(
pub async fn get_hub_app_by_id(
Authed { email, .. }: Authed,
Path(id): Path<i32>,
Extension(http_client): Extension<Client>,
) -> JsonResult<serde_json::Value> {
let value = http_get_from_hub(
http_client,
&HTTP_CLIENT,
&format!("https://hub.windmill.dev/apps/{id}/json"),
&email,
false,
backend/windmill-api/src/flows.rs
+4 -8
View File
@@ -7,7 +7,6 @@
*/
use hyper::StatusCode;
use reqwest::Client;
use sql_builder::prelude::*;
use axum::{
@@ -33,6 +32,7 @@ use crate::{
schedule::clear_schedule,
users::{require_owner_of_path, Authed},
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
pub fn workspaced_service() -> Router {
@@ -111,12 +111,9 @@ async fn list_flows(
Ok(Json(rows))
}
async fn list_hub_flows(
Authed { email, .. }: Authed,
Extension(http_client): Extension<Client>,
) -> JsonResult<serde_json::Value> {
async fn list_hub_flows(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
let flows = list_elems_from_hub(
http_client,
&HTTP_CLIENT,
"https://hub.windmill.dev/searchFlowData?approved=true",
&email,
)
@@ -145,10 +142,9 @@ async fn list_paths(
pub async fn get_hub_flow_by_id(
Authed { email, .. }: Authed,
Path(id): Path<i32>,
Extension(http_client): Extension<Client>,
) -> JsonResult<serde_json::Value> {
let value = http_get_from_hub(
http_client,
&HTTP_CLIENT,
&format!("https://hub.windmill.dev/flows/{id}/json"),
&email,
false,
backend/windmill-api/src/jobs.rs
+17 -13
View File
@@ -6,8 +6,6 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::sync::Arc;
use anyhow::Context;
use axum::{
extract::{FromRequest, Json, Path, Query},
@@ -38,7 +36,7 @@ use crate::{
db::{UserDB, DB},
users::{require_owner_of_path, Authed},
variables::get_workspace_key,
BaseUrl, QueueLimitWaitResult, TimeoutWaitResult,
BASE_URL,
};
pub fn workspaced_service() -> Router {
@@ -925,16 +923,16 @@ pub async fn get_resume_urls(
Extension(user_db): Extension<UserDB>,
Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
Query(approver): Query<QueryApprover>,
Extension(base_url): Extension<Arc<BaseUrl>>,
) -> error::JsonResult<ResumeUrls> {
let key = get_workspace_key(&w_id, &mut user_db.begin(&authed).await?).await?;
let signature = create_signature(key, job_id, resume_id, approver.approver.clone())?;
let base_url = base_url.0.clone();
let approver = approver
.approver
.as_ref()
.map(|x| format!("?approver={}", encode(x)))
.unwrap_or_else(String::new);
let base_url = BASE_URL.as_str();
let res = ResumeUrls {
approvalPage: format!(
"{base_url}/approve/{w_id}/{job_id}/{resume_id}/{signature}{approver}"
@@ -1315,18 +1313,26 @@ pub async fn check_queue_too_long(db: DB, queue_limit: Option<i64>) -> error::Re
}
Ok(())
}
lazy_static::lazy_static! {
pub static ref QUEUE_LIMIT_WAIT_RESULT: Option<i64> = std::env::var("QUEUE_LIMIT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse().ok());
pub static ref TIMEOUT_WAIT_RESULT: i32 = std::env::var("TIMEOUT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(20);
}
pub async fn run_wait_result_job_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Extension(timeout): Extension<Arc<TimeoutWaitResult>>,
Extension(queue_limit): Extension<Arc<QueueLimitWaitResult>>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
) -> error::JsonResult<serde_json::Value> {
check_queue_too_long(db, queue_limit.0.or(run_query.queue_limit)).await?;
check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
let mut tx = user_db.clone().begin(&authed).await?;
let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
@@ -1356,7 +1362,7 @@ pub async fn run_wait_result_job_by_path(
run_wait_result(
authed,
Extension(user_db),
timeout.0,
*TIMEOUT_WAIT_RESULT,
uuid,
Path((w_id, script_path)),
)
@@ -1367,7 +1373,6 @@ pub async fn run_wait_result_job_by_hash(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Extension(timeout): Extension<Arc<TimeoutWaitResult>>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
@@ -1403,7 +1408,7 @@ pub async fn run_wait_result_job_by_hash(
run_wait_result(
authed,
Extension(user_db),
timeout.0,
*TIMEOUT_WAIT_RESULT,
uuid,
Path((w_id, script_hash)),
)
@@ -1414,7 +1419,6 @@ pub async fn run_wait_result_flow_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Extension(timeout): Extension<Arc<TimeoutWaitResult>>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
@@ -1450,7 +1454,7 @@ pub async fn run_wait_result_flow_by_path(
run_wait_result(
authed,
Extension(user_db),
timeout.0,
*TIMEOUT_WAIT_RESULT,
uuid,
Path((w_id, flow_path)),
)
backend/windmill-api/src/lib.rs
+27 -54
View File
@@ -6,15 +6,17 @@
* LICENSE-AGPL for a copy of the license.
*/
use crate::oauth2::AllClients;
use argon2::Argon2;
use axum::{middleware::from_extractor, routing::get, Extension, Router};
use db::DB;
use git_version::git_version;
use reqwest::Client;
use std::{net::SocketAddr, sync::Arc};
use tower::ServiceBuilder;
use tower_cookies::CookieManagerLayer;
use tower_http::trace::TraceLayer;
use windmill_common::{error::to_anyhow, utils::rd_string};
use windmill_common::utils::rd_string;
use crate::{
db::UserDB,
@@ -50,20 +52,32 @@ mod workspaces;
pub const GIT_VERSION: &str =
git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
pub struct BaseUrl(String);
pub struct IsSecure(bool);
pub struct CookieDomain(Option<String>);
pub struct CloudHosted(bool);
pub struct ContentSecurityPolicy(String);
pub struct TimeoutWaitResult(i32);
pub struct QueueLimitWaitResult(Option<i64>);
pub use users::delete_expired_items_perdiodically;
lazy_static::lazy_static! {
pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
pub static ref SLACK_SIGNING_SECRET: Option<SlackVerifier> = std::env::var("SLACK_SIGNING_SECRET")
.ok()
.map(|x| SlackVerifier::new(x).unwrap());
static ref IS_SECURE: bool = BASE_URL.starts_with("https://");
pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build().unwrap();
pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
.map_err(|e| tracing::error!("Error building oauth clients: {}", e))
.unwrap();
}
pub async fn run_server(
db: DB,
addr: SocketAddr,
base_url: String,
mut rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<()> {
let user_db = UserDB::new(db.clone());
@@ -73,16 +87,7 @@ pub async fn run_server(
std::env::var("SUPERADMIN_SECRET").ok(),
));
let argon2 = Arc::new(Argon2::default());
let basic_clients = Arc::new(build_oauth_clients(&base_url).await?);
let slack_verifier = Arc::new(
std::env::var("SLACK_SIGNING_SECRET")
.ok()
.map(|x| SlackVerifier::new(x).unwrap()),
);
let http_client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?;
let middleware_stack = ServiceBuilder::new()
.layer(
TraceLayer::new_for_http()
@@ -93,21 +98,6 @@ pub async fn run_server(
.layer(Extension(db.clone()))
.layer(Extension(user_db))
.layer(Extension(auth_cache.clone()))
.layer(Extension(basic_clients))
.layer(Extension(Arc::new(BaseUrl(base_url.to_string()))))
.layer(Extension(Arc::new(ContentSecurityPolicy(
std::env::var("SERVE_CSP").unwrap_or("".to_owned()),
))))
.layer(Extension(Arc::new(CloudHosted(
std::env::var("CLOUD_HOSTED").is_ok(),
))))
.layer(Extension(Arc::new(IsSecure(
base_url.starts_with("https://"),
))))
.layer(Extension(Arc::new(CookieDomain(
std::env::var("COOKIE_DOMAIN").ok(),
))))
.layer(Extension(http_client))
.layer(CookieManagerLayer::new())
.layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())));
// build our application with a route
@@ -119,21 +109,7 @@ pub async fn run_server(
"/w/:workspace_id",
Router::new()
.nest("/scripts", scripts::workspaced_service())
.nest(
"/jobs",
jobs::workspaced_service()
.layer(Extension(Arc::new(TimeoutWaitResult(
std::env::var("TIMEOUT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse().ok())
.unwrap_or(20),
))))
.layer(Extension(Arc::new(QueueLimitWaitResult(
std::env::var("QUEUE_LIMIT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse().ok()),
)))),
)
.nest("/jobs", jobs::workspaced_service())
.nest(
"/users",
users::workspaced_service().layer(Extension(argon2.clone())),
@@ -174,10 +150,7 @@ pub async fn run_server(
"/auth",
users::make_unauthed_service().layer(Extension(argon2)),
)
.nest(
"/oauth",
oauth2::global_service().layer(Extension(slack_verifier)),
)
.nest("/oauth", oauth2::global_service())
.route("/version", get(git_v))
.route("/openapi.yaml", get(openapi)),
)
backend/windmill-api/src/main.rs
-70
View File
@@ -1,70 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::net::SocketAddr;
use anyhow::Ok;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
windmill_common::tracing_init::initialize_tracing();
let db = windmill_common::connect_db(true).await?;
let num_workers = std::env::var("NUM_WORKERS")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill_common::DEFAULT_NUM_WORKERS as i32);
let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
.ok()
.map(|s| {
s.parse::<bool>()
.map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
.or_else(|_| s.parse::<SocketAddr>().map(Some))
})
.transpose()?
.flatten();
let server_mode = !std::env::var("DISABLE_SERVER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
if server_mode {
windmill_api::migrate_db(&db).await?;
}
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
let shutdown_signal = windmill_common::shutdown_signal(tx);
let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
if server_mode || num_workers > 0 {
let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
let server_f = async {
if server_mode {
windmill_api::run_server(db.clone(), addr, base_url, rx.resubscribe()).await?;
}
Ok(()) as anyhow::Result<()>
};
let metrics_f = async {
match metrics_addr {
Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
.await
.map_err(anyhow::Error::from),
None => Ok(()),
}
};
futures::try_join!(shutdown_signal, server_f, metrics_f)?;
}
Ok(())
}
backend/windmill-api/src/main3.rs
+69
View File
@@ -0,0 +1,69 @@
// /*
// * Author: Ruben Fiszel
// * Copyright: Windmill Labs, Inc 2022
// * This file and its contents are licensed under the AGPLv3 License.
// * Please see the included NOTICE for copyright information and
// * LICENSE-AGPL for a copy of the license.
// */
// use std::net::SocketAddr;
// use anyhow::Ok;
// pub const DEFAULT_NUM_WORKERS: usize = 3;
// #[tokio::main]
// async fn main() -> anyhow::Result<()> {
// windmill_common::tracing_init::initialize_tracing();
// let db = windmill_common::connect_db(true).await?;
// let num_workers = std::env::var("NUM_WORKERS")
// .ok()
// .and_then(|x| x.parse::<i32>().ok())
// .unwrap_or(DEFAULT_NUM_WORKERS as i32);
// let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
// .ok()
// .map(|s| {
// s.parse::<bool>()
// .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
// .or_else(|_| s.parse::<SocketAddr>().map(Some))
// })
// .transpose()?
// .flatten();
// let server_mode = !std::env::var("DISABLE_SERVER")
// .ok()
// .and_then(|x| x.parse::<bool>().ok())
// .unwrap_or(false);
// if server_mode {
// windmill_api::migrate_db(&db).await?;
// }
// let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
// let shutdown_signal = windmill_common::shutdown_signal(tx);
// if server_mode || num_workers > 0 {
// let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
// let server_f = async {
// if server_mode {
// windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
// }
// Ok(()) as anyhow::Result<()>
// };
// let metrics_f = async {
// match metrics_addr {
// Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
// .await
// .map_err(anyhow::Error::from),
// None => Ok(()),
// }
// };
// futures::try_join!(shutdown_signal, server_f, metrics_f)?;
// }
// Ok(())
// }
backend/windmill-api/src/oauth2.rs
+37 -92
View File
@@ -8,8 +8,6 @@
use std::{collections::HashMap, fmt::Debug};
use std::sync::Arc;
use anyhow::Context;
use axum::extract::FromRequestParts;
use axum::http::request::Parts;
@@ -29,7 +27,6 @@ use oauth2::{Client as OClient, *};
use reqwest::Client;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sqlx::{Postgres, Transaction};
use tokio::{fs::File, io::AsyncReadExt};
use tower_cookies::{Cookie, Cookies};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::users::username_to_permissioned_as;
@@ -41,15 +38,14 @@ use crate::{
db::{UserDB, DB},
variables::{build_crypt, encrypt},
workspaces::WorkspaceSettings,
BaseUrl,
};
use crate::{CookieDomain, IsSecure};
use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRET};
use windmill_common::error::{self, to_anyhow, Error};
use windmill_common::oauth2::*;
use windmill_queue::JobPayload;
use std::str;
use std::{fs, str};
pub fn global_service() -> Router {
Router::new()
@@ -112,7 +108,7 @@ pub struct AllClients {
pub slack: Option<OClient>,
}
pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
let connect_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
"../../oauth_connect.json"
))?;
@@ -120,14 +116,12 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
"../../oauth_login.json"
))?;
let mut content = String::new();
let path = "./oauth.json";
if std::path::Path::new(path).exists() {
let mut file = File::open(path).await?;
file.read_to_string(&mut content).await?;
let content = if std::path::Path::new(path).exists() {
fs::read_to_string(path).map_err(to_anyhow)?
} else {
content.push_str("{}");
}
"{}".to_string()
};
let oauths: HashMap<String, OAuthClient> =
match serde_json::from_str::<HashMap<String, OAuthClient>>(&content) {
@@ -289,12 +283,10 @@ pub struct SlackBotToken {
async fn connect(
Path(client_name): Path<String>,
Query(query): Query<HashMap<String, String>>,
Extension(clients): Extension<Arc<AllClients>>,
Extension(is_secure): Extension<Arc<IsSecure>>,
cookies: Cookies,
) -> error::Result<Redirect> {
let mut query = query.clone();
let connects = &clients.connects;
let connects = &OAUTH_CLIENTS.connects;
let scopes = query
.get("scopes")
.map(|x| x.split('+').map(|x| x.to_owned()).collect());
@@ -310,7 +302,7 @@ async fn connect(
cookies,
scopes,
extra_params,
is_secure.0,
*IS_SECURE,
)
}
@@ -377,11 +369,9 @@ async fn delete_account(
Ok(format!("Deleted account id {id}"))
}
async fn list_logins(
Extension(clients): Extension<Arc<AllClients>>,
) -> error::JsonResult<Vec<String>> {
async fn list_logins() -> error::JsonResult<Vec<String>> {
Ok(Json(
clients
OAUTH_CLIENTS
.logins
.keys()
.map(|x| x.to_owned())
@@ -394,11 +384,9 @@ struct ScopesAndParams {
scopes: Vec<String>,
extra_params: Option<HashMap<String, String>>,
}
async fn list_connects(
Extension(clients): Extension<Arc<AllClients>>,
) -> error::JsonResult<HashMap<String, ScopesAndParams>> {
async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>> {
Ok(Json(
(&clients.connects)
(&OAUTH_CLIENTS.connects)
.into_iter()
.map(|(k, v)| {
(
@@ -413,12 +401,8 @@ async fn list_connects(
))
}
async fn connect_slack(
Extension(clients): Extension<Arc<AllClients>>,
Extension(is_secure): Extension<Arc<IsSecure>>,
cookies: Cookies,
) -> error::Result<Redirect> {
let mut client = clients
async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
let mut client = OAUTH_CLIENTS
.slack
.as_ref()
.ok_or_else(|| error::Error::BadRequest("slack client not setup".to_string()))?
@@ -429,7 +413,7 @@ async fn connect_slack(
client.add_scope("commands");
let url = client.authorize_url(&state);
set_cookie(&state, cookies, is_secure.0);
set_cookie(&state, cookies, *IS_SECURE);
Ok(Redirect::to(url.as_str()))
}
@@ -471,14 +455,9 @@ async fn disconnect_slack(
Ok(format!("slack disconnected"))
}
async fn login(
Extension(clients): Extension<Arc<AllClients>>,
Extension(is_secure): Extension<Arc<IsSecure>>,
Path(client_name): Path<String>,
cookies: Cookies,
) -> error::Result<Redirect> {
let clients = &clients.logins;
oauth_redirect(clients, client_name, cookies, None, None, is_secure.0)
async fn login(Path(client_name): Path<String>, cookies: Cookies) -> error::Result<Redirect> {
let clients = &OAUTH_CLIENTS.logins;
oauth_redirect(clients, client_name, cookies, None, None, *IS_SECURE)
}
#[derive(Deserialize)]
@@ -489,13 +468,11 @@ async fn refresh_token(
authed: Authed,
Path((w_id, id)): Path<(String, i32)>,
Extension(user_db): Extension<UserDB>,
Extension(clients): Extension<Arc<AllClients>>,
Extension(http_client): Extension<Client>,
Json(VariablePath { path }): Json<VariablePath>,
) -> error::Result<String> {
let tx = user_db.begin(&authed).await?;
_refresh_token(tx, &path, w_id, id, clients, http_client).await?;
_refresh_token(tx, &path, w_id, id).await?;
Ok(format!("Token at path {path} refreshed"))
}
@@ -505,8 +482,6 @@ pub async fn _refresh_token<'c>(
path: &str,
w_id: String,
id: i32,
clients: Arc<AllClients>,
http_client: Client,
) -> error::Result<String> {
let account = sqlx::query!(
"SELECT client, refresh_token FROM account WHERE workspace_id = $1 AND id = $2",
@@ -516,14 +491,14 @@ pub async fn _refresh_token<'c>(
.fetch_optional(&mut tx)
.await?;
let account = not_found_if_none(account, "Account", &id.to_string())?;
let client = (&clients
let client = (&OAUTH_CLIENTS
.connects
.get(&account.client)
.ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
.client)
.to_owned();
let token = _exchange_token(client, &account.refresh_token, http_client).await;
let token = _exchange_token(client, &account.refresh_token).await;
if let Err(token_err) = token {
sqlx::query!(
@@ -581,14 +556,10 @@ pub async fn _refresh_token<'c>(
Ok(token_str)
}
async fn _exchange_token(
client: OClient,
refresh_token: &str,
http_client: Client,
) -> Result<TokenResponse, Error> {
async fn _exchange_token(client: OClient, refresh_token: &str) -> Result<TokenResponse, Error> {
let token_json = client
.exchange_refresh_token(&RefreshToken::from(refresh_token.clone()))
.with_client(&http_client)
.with_client(&HTTP_CLIENT)
.execute::<serde_json::Value>()
.await
.map_err(to_anyhow)?;
@@ -609,11 +580,9 @@ pub struct OAuthCallback {
async fn connect_callback(
cookies: Cookies,
Path(client_name): Path<String>,
Extension(clients): Extension<Arc<AllClients>>,
Extension(http_client): Extension<Client>,
Json(callback): Json<OAuthCallback>,
) -> error::JsonResult<TokenResponse> {
let client_w_scopes = &clients
let client_w_scopes = OAUTH_CLIENTS
.connects
.get(&client_name)
.ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
@@ -621,7 +590,7 @@ async fn connect_callback(
let client = client_w_scopes.client.to_owned();
let extra_params = client_w_scopes.extra_params_callback.clone();
let token_response =
exchange_code::<TokenResponse>(callback, &cookies, client, &http_client, extra_params)
exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, extra_params)
.await?;
Ok(Json(token_response))
@@ -632,17 +601,15 @@ async fn connect_slack_callback(
authed: Authed,
cookies: Cookies,
Extension(user_db): Extension<UserDB>,
Extension(clients): Extension<Arc<AllClients>>,
Extension(http_client): Extension<Client>,
Json(callback): Json<OAuthCallback>,
) -> error::Result<String> {
let client = clients
let client = OAUTH_CLIENTS
.slack
.as_ref()
.ok_or_else(|| error::Error::BadRequest("slack client not setup".to_string()))?
.to_owned();
let token =
exchange_code::<SlackTokenResponse>(callback, &cookies, client, &http_client, None).await?;
exchange_code::<SlackTokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await?;
let mut tx = user_db.begin(&authed).await?;
@@ -760,16 +727,14 @@ where
async fn slack_command(
SlackSig { sig, ts }: SlackSig,
Extension(slack_verifier): Extension<Arc<Option<SlackVerifier>>>,
Extension(db): Extension<DB>,
Extension(base_url): Extension<Arc<BaseUrl>>,
body: Bytes,
) -> error::Result<String> {
let form: SlackCommand = serde_urlencoded::from_bytes(&body)
.map_err(|_| error::Error::BadRequest("invalid payload".to_string()))?;
let body = String::from_utf8_lossy(&body);
if slack_verifier
if SLACK_SIGNING_SECRET
.as_ref()
.as_ref()
.map(|sv| sv.verify(&ts, &body, &sig).ok())
@@ -827,7 +792,7 @@ async fn slack_command(
)
.await?;
tx.commit().await?;
let url = base_url.0.to_owned();
let url = BASE_URL.to_owned();
return Ok(format!(
"Job launched. See details at {url}/run/{uuid}?workspace={}",
&settings.workspace_id
@@ -852,31 +817,27 @@ pub struct UserInfo {
async fn login_callback(
Path(client_name): Path<String>,
cookies: Cookies,
Extension(clients): Extension<Arc<AllClients>>,
Extension(db): Extension<DB>,
Extension(http_client): Extension<Client>,
Extension(is_secure): Extension<Arc<IsSecure>>,
Extension(cookie_domain): Extension<Arc<CookieDomain>>,
Json(callback): Json<OAuthCallback>,
) -> error::Result<String> {
let client_w_config = &clients
let client_w_config = &OAUTH_CLIENTS
.logins
.get(&client_name)
.ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
let client = client_w_config.client.to_owned();
let token_res =
exchange_code::<TokenResponse>(callback, &cookies, client, &http_client, None).await;
exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await;
if let Ok(token) = token_res {
let token = &token.access_token.to_string();
let userinfo_url = client_w_config.userinfo_url.as_ref().ok_or_else(|| {
Error::BadConfig(format!("Missing userinfo_url in client {client_name}"))
})?;
let user = http_get_user_info::<UserInfo>(&http_client, userinfo_url, token).await?;
let user = http_get_user_info::<UserInfo>(&HTTP_CLIENT, userinfo_url, token).await?;
let email = match client_name.as_str() {
"github" => http_get_user_info::<Vec<GHEmailInfo>>(
&http_client,
&HTTP_CLIENT,
"https://api.github.com/user/emails",
token,
)
@@ -912,15 +873,7 @@ async fn login_callback(
if let Some((email, login_type, super_admin)) = login {
let login_type = serde_json::json!(login_type);
if login_type == client_name {
crate::users::create_session_token(
&email,
super_admin,
&mut tx,
cookies,
is_secure.0,
&cookie_domain.as_ref().0,
)
.await?;
crate::users::create_session_token(&email, super_admin, &mut tx, cookies).await?;
} else {
return Err(error::Error::BadRequest(format!(
"an user with the email associated to this login exists but with a different \
@@ -955,15 +908,7 @@ async fn login_callback(
tx.commit().await?;
invite_user_to_all_auto_invite_worspaces(&db, &email).await?;
tx = db.begin().await?;
crate::users::create_session_token(
&email,
false,
&mut tx,
cookies,
is_secure.0,
&cookie_domain.as_ref().0,
)
.await?;
crate::users::create_session_token(&email, false, &mut tx, cookies).await?;
audit_log(
&mut tx,
&email,
@@ -998,7 +943,7 @@ async fn login_callback(
tx.commit().await?;
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
let _ = http_client
let _ = HTTP_CLIENT
.post(&new_user_webhook)
.json(&serde_json::json!({"email" : &email, "event": "oauth_signup"}))
.send()
backend/windmill-api/src/scripts.rs
+6 -14
View File
@@ -6,7 +6,6 @@
* LICENSE-AGPL for a copy of the license.
*/
use reqwest::Client;
use sql_builder::prelude::*;
use windmill_audit::{audit_log, ActionKind};
@@ -15,6 +14,7 @@ use crate::{
schedule::clear_schedule,
users::{require_owner_of_path, Authed},
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
use axum::{
extract::{Extension, Path, Query},
@@ -163,12 +163,9 @@ async fn list_scripts(
Ok(Json(rows))
}
async fn list_hub_scripts(
Authed { email, .. }: Authed,
Extension(http_client): Extension<Client>,
) -> JsonResult<serde_json::Value> {
async fn list_hub_scripts(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
let asks = list_elems_from_hub(
http_client,
&HTTP_CLIENT,
"https://hub.windmill.dev/searchData?approved=true",
&email,
)
@@ -442,21 +439,16 @@ async fn create_script(
Ok((StatusCode::CREATED, format!("{}", hash)))
}
pub async fn get_hub_script_by_path(
authed: Authed,
Path(path): Path<StripPath>,
Extension(http_client): Extension<Client>,
) -> Result<String> {
windmill_common::scripts::get_hub_script_by_path(&authed.email, path, http_client).await
pub async fn get_hub_script_by_path(authed: Authed, Path(path): Path<StripPath>) -> Result<String> {
windmill_common::scripts::get_hub_script_by_path(&authed.email, path, &HTTP_CLIENT).await
}
pub async fn get_full_hub_script_by_path(
Authed { email, .. }: Authed,
Path(path): Path<StripPath>,
Extension(http_client): Extension<Client>,
) -> JsonResult<HubScript> {
Ok(Json(
windmill_common::scripts::get_full_hub_script_by_path(&email, path, http_client).await?,
windmill_common::scripts::get_full_hub_script_by_path(&email, path, &HTTP_CLIENT).await?,
))
}
backend/windmill-api/src/static_assets.rs
+13 -44
View File
@@ -9,58 +9,41 @@
use axum::{
body::{self, BoxBody},
extract::OriginalUri,
http::{header, response::Builder, Response},
http::{header, Response},
response::IntoResponse,
Extension,
};
use crate::{CloudHosted, ContentSecurityPolicy, IsSecure};
use hyper::Uri;
use mime_guess::mime;
use rust_embed::RustEmbed;
use std::sync::Arc;
// static_handler is a handler that serves static files from the
pub async fn static_handler(
Extension(is_secure): Extension<Arc<IsSecure>>,
Extension(is_cloud_hosted): Extension<Arc<CloudHosted>>,
Extension(csp): Extension<Arc<ContentSecurityPolicy>>,
OriginalUri(original_uri): OriginalUri,
) -> StaticFile {
let path = original_uri.path().trim_start_matches('/').to_string();
StaticFile(path, is_secure.0, is_cloud_hosted.0, csp)
pub async fn static_handler(OriginalUri(original_uri): OriginalUri) -> StaticFile {
StaticFile(original_uri)
}
#[derive(RustEmbed)]
#[folder = "../../frontend/build/"]
struct Asset;
pub struct StaticFile(
pub String,
pub bool,
pub bool,
pub Arc<ContentSecurityPolicy>,
);
pub struct StaticFile(Uri);
impl IntoResponse for StaticFile {
fn into_response(self) -> Response<BoxBody> {
let path = self.0;
let can_set_security_headers = self.1 && self.2;
let csp = self.3;
serve_path(path, can_set_security_headers, csp)
let path = self.0.path().trim_start_matches('/');
serve_path(path)
}
}
fn serve_path(
path: String,
can_set_security_headers: bool,
csp: Arc<ContentSecurityPolicy>,
) -> Response<BoxBody> {
const TWO_HUNDRED: &str = "200.html";
fn serve_path(path: &str) -> Response<BoxBody> {
if path.starts_with("api/") {
return Response::builder()
.status(404)
.body(body::boxed(body::Empty::new()))
.unwrap();
}
match Asset::get(path.as_str()) {
match Asset::get(path) {
Some(content) => {
let body = body::boxed(body::Full::from(content.data));
let mime = mime_guess::from_path(path).first_or_octet_stream();
@@ -75,26 +58,12 @@ fn serve_path(
res = res.header(header::CACHE_CONTROL, "no-cache, no-store, must-revalidate");
}
if can_set_security_headers {
res = set_security_headers(res, csp);
}
res.body(body).unwrap()
}
None if path.as_str().starts_with("_app/") => Response::builder()
None if path.starts_with("_app/") => Response::builder()
.status(404)
.body(body::boxed(body::Empty::new()))
.unwrap(),
None => serve_path("200.html".to_owned(), can_set_security_headers, csp),
None => serve_path(TWO_HUNDRED),
}
}
fn set_security_headers(mut res: Builder, csp: Arc<ContentSecurityPolicy>) -> Builder {
res = res.header("X-Frame-Options", "DENY");
res = res.header("X-Content-Type-Options", "nosniff");
if !csp.0.is_empty() {
res = res.header("Content-Security-Policy", &csp.0);
}
res
}
backend/windmill-api/src/users.rs
+9 -22
View File
@@ -13,7 +13,7 @@ use crate::{
folders::get_folders_for_user,
utils::require_super_admin,
workspaces::invite_user_to_all_auto_invite_worspaces,
CookieDomain, IsSecure,
COOKIE_DOMAIN, IS_SECURE,
};
use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
use axum::{
@@ -721,14 +721,12 @@ async fn logout(
Tokened { token }: Tokened,
cookies: Cookies,
Extension(db): Extension<DB>,
Extension(cookie_domain): Extension<Arc<CookieDomain>>,
Query(LogoutQuery { rd }): Query<LogoutQuery>,
) -> Result<Response> {
let mut cookie = Cookie::new(COOKIE_NAME, "");
cookie.set_path(COOKIE_PATH);
let domain = cookie_domain.0.clone();
if domain.is_some() {
cookie.set_domain(domain.clone().unwrap());
if COOKIE_DOMAIN.is_some() {
cookie.set_domain(COOKIE_DOMAIN.clone().unwrap());
}
cookies.remove(cookie);
let mut tx = db.begin().await?;
@@ -1565,8 +1563,6 @@ async fn login(
cookies: Cookies,
Extension(db): Extension<DB>,
Extension(argon2): Extension<Arc<Argon2<'_>>>,
Extension(is_secure): Extension<Arc<IsSecure>>,
Extension(cookie_domain): Extension<Arc<CookieDomain>>,
Json(Login { email, password }): Json<Login>,
) -> Result<String> {
let mut tx = db.begin().await?;
@@ -1596,7 +1592,7 @@ async fn login(
.execute(&mut tx)
.await?;
let mut c = Cookie::new("first_time", "1");
if let Some(domain) = cookie_domain.as_ref().0.clone() {
if let Some(domain) = COOKIE_DOMAIN.as_ref() {
c.set_domain(domain);
}
c.set_secure(false);
@@ -1607,15 +1603,8 @@ async fn login(
cookies.add(c);
}
let token = create_session_token(
&email,
super_admin,
&mut tx,
cookies,
is_secure.0,
&cookie_domain.as_ref().0,
)
.await?;
let token = create_session_token(&email, super_admin, &mut tx, cookies).await?;
tx.commit().await?;
Ok(token)
}
@@ -1629,8 +1618,6 @@ pub async fn create_session_token<'c>(
super_admin: bool,
tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
cookies: Cookies,
is_secure: bool,
domain: &Option<String>,
) -> Result<String> {
let token = rd_string(30);
sqlx::query!(
@@ -1646,12 +1633,12 @@ pub async fn create_session_token<'c>(
.execute(tx)
.await?;
let mut cookie = Cookie::new(COOKIE_NAME, token.clone());
cookie.set_secure(is_secure);
cookie.set_secure(*IS_SECURE);
cookie.set_same_site(Some(cookie::SameSite::Lax));
cookie.set_http_only(true);
cookie.set_path(COOKIE_PATH);
if domain.is_some() {
cookie.set_domain(domain.clone().unwrap());
if COOKIE_DOMAIN.is_some() {
cookie.set_domain(COOKIE_DOMAIN.clone().unwrap());
}
let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc();
expire += time::Duration::days(3);
backend/windmill-api/src/variables.rs
+2 -20
View File
@@ -6,14 +6,11 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::sync::Arc;
use crate::{
db::{UserDB, DB},
oauth2::{AllClients, _refresh_token},
oauth2::_refresh_token,
users::{require_owner_of_path, Authed},
webhook_util::{WebhookMessage, WebhookShared},
BaseUrl,
};
/*
* Author: Ruben Fiszel
@@ -37,7 +34,6 @@ use windmill_common::{
};
use magic_crypt::{MagicCrypt256, MagicCryptTrait};
use reqwest::Client;
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
@@ -54,7 +50,6 @@ pub fn workspaced_service() -> Router {
async fn list_contextual_variables(
Path(w_id): Path<String>,
Extension(base_url): Extension<Arc<BaseUrl>>,
Authed { username, email, .. }: Authed,
) -> JsonResult<Vec<ContextualVariable>> {
Ok(Json(
@@ -65,7 +60,6 @@ async fn list_contextual_variables(
&username,
"017e0ad5-f499-73b6-5488-92a61c5196dd",
format!("u/{username}").as_str(),
&base_url.0,
Some("u/user/script_path".to_string()),
Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()),
Some("u/user/encapsulating_flow_path".to_string()),
@@ -111,8 +105,6 @@ async fn get_variable(
Extension(user_db): Extension<UserDB>,
Query(q): Query<GetVariableQuery>,
Path((w_id, path)): Path<(String, StripPath)>,
Extension(clients): Extension<Arc<AllClients>>,
Extension(http_client): Extension<Client>,
) -> JsonResult<ListableVariable> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
@@ -151,17 +143,7 @@ async fn get_variable(
let value = variable.value.unwrap_or_else(|| "".to_string());
ListableVariable {
value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
Some(
_refresh_token(
tx,
&variable.path,
w_id,
variable.account.unwrap(),
clients,
http_client,
)
.await?,
)
Some(_refresh_token(tx, &variable.path, w_id, variable.account.unwrap()).await?)
} else if !value.is_empty() && decrypt_secret {
let mc = build_crypt(&mut tx, &w_id).await?;
tx.commit().await?;
backend/windmill-api/src/workspaces.rs
+6 -10
View File
@@ -6,7 +6,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::{str::FromStr, sync::Arc};
use std::str::FromStr;
use crate::{
db::{UserDB, DB},
@@ -14,7 +14,7 @@ use crate::{
resources::{Resource, ResourceType},
users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK},
utils::require_super_admin,
BaseUrl,
BASE_URL, HTTP_CLIENT,
};
use axum::{
body::StreamBody,
@@ -24,7 +24,6 @@ use axum::{
routing::{delete, get, post},
Json, Router,
};
use reqwest::Client;
use stripe::CustomerId;
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
@@ -226,15 +225,14 @@ async fn stripe_checkout(
authed: Authed,
Path(w_id): Path<String>,
Query(plan): Query<PlanQuery>,
Extension(base_url): Extension<Arc<BaseUrl>>,
) -> Result<Redirect> {
// #[cfg(feature = "enterprise")]
{
require_admin(authed.is_admin, &authed.username)?;
let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
let success_rd = format!("{}/workspace_settings/checkout?success=true", base_url.0);
let failure_rd = format!("{}/workspace_settings/checkout?success=false", base_url.0);
let success_rd = format!("{}/workspace_settings/checkout?success=true", *BASE_URL);
let failure_rd = format!("{}/workspace_settings/checkout?success=false", *BASE_URL);
let checkout_session = {
let mut params = stripe::CreateCheckoutSession::new(&failure_rd, &success_rd);
params.mode = Some(stripe::CheckoutSessionMode::Subscription);
@@ -292,7 +290,6 @@ async fn stripe_portal(
authed: Authed,
Path(w_id): Path<String>,
Extension(db): Extension<DB>,
Extension(base_url): Extension<Arc<BaseUrl>>,
) -> Result<Redirect> {
require_admin(authed.is_admin, &authed.username)?;
let customer_id = sqlx::query_scalar!(
@@ -303,7 +300,7 @@ async fn stripe_portal(
.await?
.ok_or_else(|| Error::InternalErr(format!("no customer id for workspace {}", w_id)))?;
let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
let success_rd = format!("{}/workspace_settings?tab=premium", base_url.0);
let success_rd = format!("{}/workspace_settings?tab=premium", *BASE_URL);
let portal_session = {
let customer_id = CustomerId::from_str(&customer_id).unwrap();
let mut params = stripe::CreateBillingPortalSession::new(customer_id);
@@ -926,7 +923,6 @@ pub async fn invite_user_to_all_auto_invite_worspaces(db: &DB, email: &str) -> R
async fn invite_user(
Authed { username, is_admin, .. }: Authed,
Extension(db): Extension<DB>,
Extension(http_client): Extension<Client>,
Path(w_id): Path<String>,
Json(nu): Json<NewWorkspaceInvite>,
) -> Result<(StatusCode, String)> {
@@ -949,7 +945,7 @@ async fn invite_user(
tx.commit().await?;
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
let _ = http_client
let _ = &HTTP_CLIENT
.post(&new_user_webhook)
.json(&serde_json::json!({"email" : &nu.email, "event": "new_invite"}))
.send()
backend/windmill-common/Cargo.toml
+1
View File
@@ -41,3 +41,4 @@ hyper = { workspace = true, optional = true }
tokio = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
tracing-subscriber = { workspace = true, optional = true }
lazy_static.workspace = true
backend/windmill-common/src/lib.rs
+4 -3
View File
@@ -26,12 +26,13 @@ pub mod variables;
#[cfg(feature = "tracing_init")]
pub mod tracing_init;
pub const DEFAULT_NUM_WORKERS: usize = 3;
pub const DEFAULT_TIMEOUT: i32 = 300;
pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 3;
lazy_static::lazy_static! {
pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
}
#[cfg(feature = "tokio")]
pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
use std::io;
backend/windmill-common/src/scripts.rs
+2 -2
View File
@@ -210,7 +210,7 @@ pub fn to_hex_string(i: &i64) -> String {
pub async fn get_hub_script_by_path(
email: &str,
path: StripPath,
http_client: reqwest::Client,
http_client: &reqwest::Client,
) -> crate::error::Result<String> {
use crate::{
error::{to_anyhow, Error},
@@ -239,7 +239,7 @@ pub async fn get_hub_script_by_path(
pub async fn get_full_hub_script_by_path(
email: &str,
path: StripPath,
http_client: reqwest::Client,
http_client: &reqwest::Client,
) -> crate::error::Result<HubScript> {
use crate::{
error::{to_anyhow, Error},
backend/windmill-common/src/utils.rs
+2 -2
View File
@@ -74,7 +74,7 @@ pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U)
#[cfg(feature = "reqwest")]
pub async fn list_elems_from_hub(
http_client: reqwest::Client,
http_client: &reqwest::Client,
url: &str,
email: &str,
) -> Result<serde_json::Value> {
@@ -88,7 +88,7 @@ pub async fn list_elems_from_hub(
#[cfg(feature = "reqwest")]
pub async fn http_get_from_hub(
http_client: reqwest::Client,
http_client: &reqwest::Client,
url: &str,
email: &str,
plain: bool,
backend/windmill-common/src/variables.rs
+3 -2
View File
@@ -8,6 +8,8 @@
use serde::{Deserialize, Serialize};
use crate::BASE_URL;
#[derive(Serialize, Clone)]
pub struct ContextualVariable {
@@ -66,7 +68,6 @@ pub fn get_reserved_variables(
username: &str,
job_id: &str,
permissioned_as: &str,
base_url: &str,
path: Option<String>,
flow_id: Option<String>,
flow_path: Option<String>,
@@ -114,7 +115,7 @@ pub fn get_reserved_variables(
},
ContextualVariable {
name: "WM_BASE_URL".to_string(),
value: base_url.to_string(),
value: BASE_URL.clone(),
description: "base url of this instance".to_string(),
},
ContextualVariable {
backend/windmill-queue/src/jobs.rs
+12 -7
View File
@@ -9,6 +9,7 @@
use std::{collections::HashMap, str::FromStr};
use anyhow::Context;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use sqlx::{Pool, Postgres, Transaction};
use tracing::{instrument, Instrument};
@@ -16,7 +17,7 @@ use ulid::Ulid;
use uuid::Uuid;
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{self, to_anyhow, Error},
error::{self, Error},
flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
flows::{FlowModule, FlowModuleValue, FlowValue},
scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
@@ -24,6 +25,10 @@ use windmill_common::{
};
lazy_static::lazy_static! {
pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build().unwrap();
// TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens.
static ref QUEUE_PUSH_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
"queue_push_count",
@@ -406,7 +411,7 @@ pub async fn push<'c>(
)
}
JobPayload::ScriptHub { path } => {
let script = get_hub_script(path.clone(), email)
let script = get_hub_script(&HTTP_CLIENT, path.clone(), email)
.await
.context("error fetching hub script")?;
(
@@ -610,11 +615,11 @@ pub fn canceled_job_to_result(job: &QueuedJob) -> serde_json::Value {
serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler})
}
pub async fn get_hub_script(path: String, email: &str) -> error::Result<HubScript> {
let client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?;
pub async fn get_hub_script(
client: &reqwest::Client,
path: String,
email: &str,
) -> error::Result<HubScript> {
get_full_hub_script_by_path(email, StripPath(path), client)
.await
.map(|e| e)
backend/windmill-worker/Cargo.toml
+3 -4
View File
@@ -3,11 +3,10 @@ name = "windmill-worker"
version.workspace = true
authors.workspace = true
edition.workspace = true
default-run = "worker"
[[bin]]
name = "worker"
path = "./src/main.rs"
[lib]
name = "windmill_worker"
path = "src/lib.rs"
[features]
default = []
backend/windmill-worker/src/js_eval.rs
+5 -4
View File
@@ -34,10 +34,11 @@ pub async fn eval_timeout(
env: Vec<(String, serde_json::Value)>,
creds: Option<EvalCreds>,
by_id: Option<IdContext>,
base_internal_url: String,
base_internal_url: &str,
) -> anyhow::Result<serde_json::Value> {
let expr2 = expr.clone();
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
let base_internal_url: String = base_internal_url.to_string();
timeout(
std::time::Duration::from_millis(2000),
tokio::task::spawn_blocking(move || {
@@ -346,7 +347,7 @@ mod tests {
let code = "value.test + params.test";
let mut runtime = JsRuntime::new(RuntimeOptions::default());
let res = eval(&mut runtime, code, env, None, None, "").await?;
let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
assert_eq!(res, json!(4));
Ok(())
}
@@ -359,7 +360,7 @@ mod tests {
multiline template`";
let mut runtime = JsRuntime::new(RuntimeOptions::default());
let res = eval(&mut runtime, code, env, None, None, "").await?;
let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
assert_eq!(res, json!("my 5\nmultiline template"));
Ok(())
}
@@ -372,7 +373,7 @@ multiline template`";
];
let code = r#"params.test"#;
let res = eval_timeout(code.to_string(), env, None, None, "".to_string()).await?;
let res = eval_timeout(code.to_string(), env, None, None, String::new().as_str()).await?;
assert_eq!(res, json!(2));
Ok(())
}
backend/windmill-worker/src/main.rs
-138
View File
@@ -1,138 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::{net::SocketAddr, time::Duration};
use anyhow::Context;
use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
use windmill_common::{
error::{self, Error},
utils::rd_string,
};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// dotenv().ok();
windmill_common::tracing_init::initialize_tracing();
let db = async {
let database_url = std::env::var("DATABASE_URL")
.map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
Err(_) => 10,
};
Ok::<Pool<Postgres>, error::Error>(
PgPoolOptions::new()
.max_connections(max_connections)
.max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
.connect(&database_url)
.await
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
)
}
.await?;
let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
.ok()
.map(|s| {
s.parse::<bool>()
.map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
.or_else(|_| s.parse::<SocketAddr>().map(Some))
})
.transpose()?
.flatten();
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
let shutdown_signal = windmill_common::shutdown_signal(tx);
let base_internal_url =
std::env::var("BASE_INTERNAL_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
let base_url = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
let timeout = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill_common::DEFAULT_TIMEOUT);
let workers_f = async {
let sleep_queue = std::env::var("SLEEP_QUEUE")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(windmill_common::DEFAULT_SLEEP_QUEUE);
let disable_nuser = std::env::var("DISABLE_NUSER")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let disable_nsjail = std::env::var("DISABLE_NSJAIL")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(true);
let keep_job_dir = std::env::var("KEEP_JOB_DIR")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let sync_bucket = std::env::var("S3_CACHE_BUCKET")
.ok()
.map(|e| Some(e))
.unwrap_or(None);
tracing::info!(
"DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
{base_url}, SLEEP_QUEUE: {sleep_queue}, TIMEOUT: \
{timeout}, KEEP_JOB_DIR: {keep_job_dir}"
);
let instance_name = rd_string(5);
let ip = windmill_common::external_ip::get_ip()
.await
.unwrap_or_else(|e| {
tracing::warn!(error = e.to_string(), "failed to get external IP");
"unretrievable IP".to_string()
});
let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
windmill_worker::run_worker(
&db.clone(),
timeout,
&instance_name,
worker_name,
1,
1,
&ip,
sleep_queue,
windmill_worker::WorkerConfig {
disable_nsjail,
disable_nuser,
base_internal_url,
base_url,
keep_job_dir,
},
sync_bucket,
rx.resubscribe(),
)
.await;
Ok(()) as anyhow::Result<()>
};
let metrics_f = async {
match metrics_addr {
Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
.await
.map_err(anyhow::Error::from),
None => Ok(()),
}
};
futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
Ok(())
}
backend/windmill-worker/src/main2.rs
+92
View File
@@ -0,0 +1,92 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
// use std::{net::SocketAddr, time::Duration};
// use anyhow::Context;
// use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
// use windmill_common::{
// error::{self, Error},
// utils::rd_string,
// };
// #[tokio::main]
// async fn main() -> anyhow::Result<()> {
// // dotenv().ok();
// windmill_common::tracing_init::initialize_tracing();
// let db = async {
// let database_url = std::env::var("DATABASE_URL")
// .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
// let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
// Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
// Err(_) => 10,
// };
// Ok::<Pool<Postgres>, error::Error>(
// PgPoolOptions::new()
// .max_connections(max_connections)
// .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
// .connect(&database_url)
// .await
// .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
// )
// }
// .await?;
// let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
// .ok()
// .map(|s| {
// s.parse::<bool>()
// .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
// .or_else(|_| s.parse::<SocketAddr>().map(Some))
// })
// .transpose()?
// .flatten();
// let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
// let shutdown_signal = windmill_common::shutdown_signal(tx);
// let workers_f = async {
// let instance_name = rd_string(5);
// let ip = windmill_common::external_ip::get_ip()
// .await
// .unwrap_or_else(|e| {
// tracing::warn!(error = e.to_string(), "failed to get external IP");
// "unretrievable IP".to_string()
// });
// let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
// windmill_worker::run_worker(
// &db.clone(),
// &instance_name,
// worker_name,
// 1,
// 1,
// &ip,
// rx.resubscribe(),
// )
// .await;
// Ok(()) as anyhow::Result<()>
// };
// let metrics_f = async {
// match metrics_addr {
// Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
// .await
// .map_err(anyhow::Error::from),
// None => Ok(()),
// }
// };
// futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
// Ok(())
// }
backend/windmill-worker/src/worker.rs
+258 -325
View File
File diff suppressed because it is too large Load Diff
backend/windmill-worker/src/worker_flow.rs
+16 -19
View File
@@ -12,7 +12,7 @@ use std::time::Duration;
use crate::jobs::{add_completed_job, add_completed_job_error, schedule_again_if_scheduled};
use crate::js_eval::{eval_timeout, EvalCreds, IdContext};
use crate::worker;
use crate::{worker, KEEP_JOB_DIR};
use anyhow::Context;
use async_recursion::async_recursion;
use dyn_iter::DynIter;
@@ -50,9 +50,8 @@ pub async fn update_flow_status_after_job_completion(
unrecoverable: bool,
same_worker_tx: Sender<Uuid>,
worker_dir: &str,
keep_job_dir: bool,
base_internal_url: &str,
stop_early_override: Option<bool>,
base_internal_url: &str,
) -> error::Result<()> {
tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id}");
@@ -121,7 +120,7 @@ pub async fn update_flow_status_after_job_completion(
let stop_early = success
&& if let Some(expr) = r.stop_early_expr.clone() {
compute_bool_from_expr(expr, &r.args, result.clone(), base_internal_url, None, None)
compute_bool_from_expr(expr, &r.args, result.clone(), None, None, base_internal_url)
.await?
} else {
false
@@ -481,7 +480,7 @@ pub async fn update_flow_status_after_job_completion(
};
if done {
if flow_job.same_worker && !keep_job_dir {
if flow_job.same_worker && !*KEEP_JOB_DIR {
let _ = tokio::fs::remove_dir_all(format!("{worker_dir}/{}", flow_job.id)).await;
}
@@ -498,13 +497,12 @@ pub async fn update_flow_status_after_job_completion(
false,
same_worker_tx.clone(),
worker_dir,
keep_job_dir,
base_internal_url,
if stop_early {
Some(skip_if_stop_early)
} else {
None
},
base_internal_url,
)
.await?);
}
@@ -583,9 +581,9 @@ async fn compute_bool_from_expr(
expr: String,
flow_args: &Option<serde_json::Value>,
result: serde_json::Value,
base_internal_url: &str,
by_id: Option<IdContext>,
creds: Option<EvalCreds>,
base_internal_url: &str,
) -> error::Result<bool> {
let flow_input = flow_args.clone().unwrap_or_else(|| json!({}));
match eval_timeout(
@@ -598,7 +596,7 @@ async fn compute_bool_from_expr(
.into(),
creds,
by_id,
base_internal_url.to_string(),
base_internal_url,
)
.await?
{
@@ -722,7 +720,7 @@ async fn transform_input(
context,
Some(EvalCreds { workspace: workspace.to_string(), token: token.to_string() }),
Some(by_id.clone()),
base_internal_url.to_string(),
base_internal_url,
)
.await
.map_err(|e| {
@@ -769,8 +767,8 @@ pub async fn handle_flow(
client,
last_result,
same_worker_tx,
base_internal_url,
worker_dir,
base_internal_url,
)
.await?;
Ok(())
@@ -786,8 +784,8 @@ async fn push_next_flow_job(
client: &windmill_api_client::Client,
mut last_result: serde_json::Value,
same_worker_tx: Sender<Uuid>,
base_internal_url: &str,
worker_dir: &str,
base_internal_url: &str,
) -> error::Result<()> {
let mut i = usize::try_from(status.step)
.with_context(|| format!("invalid module index {}", status.step))?;
@@ -816,9 +814,8 @@ async fn push_next_flow_job(
true,
same_worker_tx,
worker_dir,
false,
base_internal_url,
None,
base_internal_url,
)
.await;
}
@@ -858,7 +855,7 @@ async fn push_next_flow_job(
.into(),
None,
None,
"".to_string(),
base_internal_url,
)
.await
.map_err(|e| {
@@ -1162,8 +1159,8 @@ async fn push_next_flow_job(
&status,
&status_module,
last_result.clone(),
base_internal_url,
previous_id,
base_internal_url,
)
.await?;
tx.commit().await?;
@@ -1504,8 +1501,8 @@ async fn compute_next_flow_transform<'c>(
status: &FlowStatus,
status_module: &FlowStatusModule,
last_result: serde_json::Value,
base_internal_url: &str,
previous_id: String,
base_internal_url: &str,
) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, NextFlowTransform)> {
match &module.value {
FlowModuleValue::Identity => Ok((
@@ -1701,12 +1698,12 @@ async fn compute_next_flow_transform<'c>(
b.expr.to_string(),
&flow_job.args,
last_result.clone(),
base_internal_url,
Some(idcontext.clone()),
Some(EvalCreds {
workspace: flow_job.workspace_id.clone(),
token: token.to_string(),
}),
base_internal_url,
)
.await?;
@@ -1920,7 +1917,7 @@ where
vars(),
Some(EvalCreds { workspace, token }),
by_id,
base_internal_url.to_string(),
base_internal_url,
)
.await
}
frontend/src/routes/(root)/(logged)/+layout.svelte
-1
View File
@@ -19,7 +19,6 @@
import FavoriteMenu from '$lib/components/sidebar/FavoriteMenu.svelte'
OpenAPI.WITH_CREDENTIALS = true
let menuOpen = false
let isCollapsed = false
let userSettings: UserSettings