chore: fasten compilation time

This commit is contained in:
Ruben Fiszel
2024-02-24 17:10:38 +01:00
parent 82c8b1d1dd
commit c3ef987e28
20 changed files with 73 additions and 415 deletions
+1 -1
View File
@@ -130,7 +130,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus
nsjail=true
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
+1
View File
@@ -45,6 +45,7 @@ loki = ["windmill-common/loki"]
pg_embed = ["dep:pg-embed"]
embedding = ["windmill-api/embedding"]
parquet = ["windmill-api/parquet", "windmill-common/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"]
[dependencies]
anyhow.workspace = true
-16
View File
@@ -1,16 +0,0 @@
use anyhow::anyhow;
#[cfg(feature = "enterprise")]
use windmill_common::error::{Error, Result};
pub async fn set_license_key(_license_key: String) -> anyhow::Result<()> {
// Implementation is not open source
Err(anyhow!("License cannot be set in Windmill CE"))
}
#[cfg(feature = "enterprise")]
pub async fn verify_license_key() -> Result<()> {
// Implementation is not open source
Err(Error::InternalErr(
"License always invalid in Windmill CE".to_string(),
))
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/src/ee.rs
+2 -2
View File
@@ -16,7 +16,7 @@ enterprise_saml = ["dep:samael"]
benchmark = []
embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
parquet = ["dep:polars", "dep:polars-io", "dep:object_store", "windmill-common/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
[dependencies]
windmill-queue.workspace = true
@@ -66,7 +66,7 @@ sha2.workspace = true
urlencoding.workspace = true
async-stripe = { workspace = true, optional = true }
lazy_static.workspace = true
prometheus.workspace = true
prometheus = { workspace = true, optional = true }
async_zip.workspace = true
rsmq_async.workspace = true
regex.workspace = true
@@ -9,9 +9,6 @@ use axum::{Extension, Json};
use axum::Router;
#[cfg(feature = "enterprise")]
#[cfg(feature = "parquet")]
use polars::prelude::IntoVec;
#[cfg(feature = "enterprise")]
use serde::Serialize;
#[cfg(feature = "enterprise")]
-6
View File
@@ -1,6 +0,0 @@
use anyhow::anyhow;
pub async fn validate_license_key(_license_key: String) -> anyhow::Result<String> {
// Implementation is not open source
Err(anyhow!("License can't be validated in Windmill CE"))
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-api/src/ee.rs
@@ -1,5 +0,0 @@
use axum::Router;
pub fn workspaced_service() -> Router {
Router::new()
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-api/src/job_helpers_ee.rs
+25 -6
View File
@@ -9,7 +9,9 @@
use axum::http::HeaderValue;
use serde_json::value::RawValue;
use std::collections::HashMap;
#[cfg(feature = "prometheus")]
use std::sync::atomic::Ordering;
#[cfg(feature = "prometheus")]
use tokio::time::Instant;
use windmill_common::flow_status::{JobResult, RestartedFrom};
use windmill_common::variables::get_workspace_key;
@@ -52,15 +54,25 @@ use windmill_common::{
users::username_to_permissioned_as,
utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
};
use windmill_common::{
get_latest_deployed_hash_for_path, BASE_URL, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
};
#[cfg(feature = "prometheus")]
use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED};
use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL};
use windmill_queue::{
add_completed_job_error, get_queued_job, get_result_by_id_from_running_flow, job_is_complete,
push, CanceledBy, PushArgs, PushIsolationLevel,
};
fn setup_list_jobs_debug_metrics() -> Option<prometheus::Histogram> {
#[cfg(feature = "prometheus")]
type Histo = prometheus::Histogram;
#[cfg(not(feature = "prometheus"))]
type Histo = ();
#[cfg(feature = "prometheus")]
fn setup_list_jobs_debug_metrics() -> Option<Histo> {
let api_list_jobs_query_duration = if METRICS_DEBUG_ENABLED.load(Ordering::Relaxed)
&& METRICS_ENABLED.load(Ordering::Relaxed)
{
@@ -78,6 +90,11 @@ fn setup_list_jobs_debug_metrics() -> Option<prometheus::Histogram> {
api_list_jobs_query_duration
}
#[cfg(not(feature = "prometheus"))]
fn setup_list_jobs_debug_metrics() -> Option<Histo> {
None
}
pub fn workspaced_service() -> Router {
let cors = CorsLayer::new()
.allow_methods([http::Method::GET, http::Method::POST])
@@ -917,7 +934,7 @@ async fn list_jobs(
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(lq): Query<ListCompletedQuery>,
Extension(api_list_jobs_query_duration): Extension<Option<prometheus::Histogram>>,
Extension(_api_list_jobs_query_duration): Extension<Option<Histo>>,
) -> error::JsonResult<Vec<Job>> {
check_scopes(&authed, || format!("listjobs"))?;
@@ -1051,12 +1068,14 @@ async fn list_jobs(
};
let mut tx = user_db.begin(&authed).await?;
#[cfg(feature = "prometheus")]
let start = Instant::now();
let jobs: Vec<UnifiedJob> = sqlx::query_as(&sql).fetch_all(&mut *tx).await?;
tx.commit().await?;
if let Some(api_list_jobs_query_duration) = api_list_jobs_query_duration {
#[cfg(feature = "prometheus")]
if let Some(api_list_jobs_query_duration) = _api_list_jobs_query_duration {
let duration = start.elapsed().as_secs_f64();
api_list_jobs_query_duration.observe(duration);
tracing::info!("list_jobs query took {}s: {}", duration, sql);
+16 -2
View File
@@ -58,7 +58,8 @@ mod granular_acls;
mod groups;
mod inputs;
mod integration;
pub mod job_helpers_ee;
#[cfg(feature = "parquet")]
mod job_helpers_ee;
pub mod job_metrics;
pub mod jobs;
pub mod oauth2_ee;
@@ -180,6 +181,19 @@ pub async fn run_server(
};
let job_helpers_service = {
#[cfg(feature = "parquet")]
{
job_helpers_ee::workspaced_service()
}
#[cfg(not(feature = "parquet"))]
{
Router::new()
}
};
// build our application with a route
let app = Router::new()
@@ -205,7 +219,7 @@ pub async fn run_server(
.nest("/groups", groups::workspaced_service())
.nest("/inputs", inputs::workspaced_service())
.nest("/job_metrics", job_metrics::workspaced_service())
.nest("/job_helpers", job_helpers_ee::workspaced_service())
.nest("/job_helpers", job_helpers_service)
.nest("/jobs", jobs::workspaced_service())
.nest("/oauth", oauth2_ee::workspaced_service())
.nest("/openai", openai::workspaced_service())
-212
View File
@@ -1,212 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::{collections::HashMap, fmt::Debug};
use axum::body::StreamBody;
use axum::response::IntoResponse;
use axum::{routing::get, Json, Router};
use hmac::Mac;
use hyper::{HeaderMap, StatusCode};
use oauth2::{Client as OClient, *};
use serde::{Deserialize, Serialize};
use sqlx::{Postgres, Transaction};
use windmill_common::more_serde::maybe_number_opt;
use crate::{HTTP_CLIENT, OAUTH_CLIENTS};
use windmill_common::error::{self, to_anyhow};
use windmill_common::oauth2::*;
use crate::db::DB;
use std::str;
pub fn global_service() -> Router {
Router::new()
.route("/list_supabase", get(list_supabase))
.route("/list_logins", get(list_logins))
.route("/list_connects", get(list_connects))
}
pub fn workspaced_service() -> Router {
Router::new()
}
#[derive(Serialize)]
#[serde(tag = "type")]
pub enum InstanceEvent {
UserAdded { email: String },
// UserDeleted { email: String },
// UserDeletedWorkspace { workspace: String, email: String },
UserAddedWorkspace { workspace: String, email: String },
UserInvitedWorkspace { workspace: String, email: String },
UserJoinedWorkspace { workspace: String, email: String, username: String },
}
#[derive(Debug, Clone)]
pub struct ClientWithScopes {
_client: OClient,
scopes: Vec<String>,
extra_params: Option<HashMap<String, String>>,
_extra_params_callback: Option<HashMap<String, String>>,
_allowed_domains: Option<Vec<String>>,
_userinfo_url: Option<String>,
}
pub type BasicClientsMap = HashMap<String, ClientWithScopes>;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct OAuthConfig {
auth_url: String,
token_url: String,
userinfo_url: Option<String>,
scopes: Option<Vec<String>>,
extra_params: Option<HashMap<String, String>>,
extra_params_callback: Option<HashMap<String, String>>,
req_body_auth: Option<bool>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct OAuthClient {
id: String,
secret: String,
allowed_domains: Option<Vec<String>>,
connect_config: Option<OAuthConfig>,
login_config: Option<OAuthConfig>,
}
#[derive(Debug)]
pub struct AllClients {
pub logins: BasicClientsMap,
pub connects: BasicClientsMap,
pub slack: Option<OClient>,
}
pub fn build_oauth_clients(
_base_url: &str,
_oauths_from_config: Option<HashMap<String, OAuthClient>>,
) -> anyhow::Result<AllClients> {
// Implementation is not open source
return Ok(AllClients {
logins: HashMap::default(),
connects: HashMap::default(),
slack: None,
});
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct TokenResponse {
access_token: AccessToken,
#[serde(deserialize_with = "maybe_number_opt")]
#[serde(default)]
expires_in: Option<u64>,
refresh_token: Option<RefreshToken>,
#[serde(deserialize_with = "helpers::deserialize_space_delimited_vec")]
#[serde(serialize_with = "helpers::serialize_space_delimited_vec")]
#[serde(default)]
scope: Option<Vec<Scope>>,
}
#[derive(Serialize)]
struct Logins {
oauth: Vec<String>,
saml: Option<String>,
}
async fn list_logins() -> error::JsonResult<Logins> {
// Implementation is not open source
return Ok(Json(Logins { oauth: vec![], saml: None }));
}
#[derive(Serialize)]
struct ScopesAndParams {
scopes: Vec<String>,
extra_params: Option<HashMap<String, String>>,
}
async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>> {
Ok(Json(
(&OAUTH_CLIENTS.read().await.connects)
.into_iter()
.map(|(k, v)| {
(
k.to_owned(),
ScopesAndParams {
scopes: v.scopes.clone(),
extra_params: v.extra_params.clone(),
},
)
})
.collect::<HashMap<String, ScopesAndParams>>(),
))
}
pub async fn _refresh_token<'c>(
_tx: Transaction<'c, Postgres>,
_path: &str,
_w_id: &str,
_id: i32,
) -> error::Result<String> {
// Implementation is not open source
Err(error::Error::BadRequest(
"Not implemented in Windmill's Open Source repository".to_string(),
))
}
async fn list_supabase(headers: HeaderMap) -> impl IntoResponse {
let token = headers
.get("X-Supabase-Token")
.map(|x| x.to_str().unwrap_or(""))
.unwrap_or("");
let resp = HTTP_CLIENT
.get("https://api.supabase.com/v1/projects")
.bearer_auth(token)
.send()
.await
.map_err(to_anyhow)?;
let status_code = resp.status();
let stream = resp.bytes_stream();
Ok((status_code, StreamBody::new(stream))) as error::Result<(StatusCode, StreamBody<_>)>
}
pub async fn check_nb_of_user(db: &DB) -> error::Result<()> {
let nb_users_sso =
sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",)
.fetch_one(db)
.await?;
if nb_users_sso.unwrap_or(0) >= 10 {
return Err(error::Error::BadRequest(
"You have reached the maximum number of oauth users accounts (10) without an enterprise license"
.to_string(),
));
}
let nb_users = sqlx::query_scalar!("SELECT COUNT(*) FROM password",)
.fetch_one(db)
.await?;
if nb_users.unwrap_or(0) >= 50 {
return Err(error::Error::BadRequest(
"You have reached the maximum number of accounts (50) without an enterprise license"
.to_string(),
));
}
return Ok(());
}
#[derive(Clone, Debug)]
pub struct SlackVerifier {
_mac: HmacSha256,
}
impl SlackVerifier {
pub fn new<S: AsRef<[u8]>>(secret: S) -> anyhow::Result<SlackVerifier> {
HmacSha256::new_from_slice(secret.as_ref())
.map(|mac| SlackVerifier { _mac: mac })
.map_err(|_| anyhow::anyhow!("invalid secret"))
}
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-api/src/oauth2_ee.rs
-17
View File
@@ -1,17 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2023
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::Router;
pub fn global_service() -> Router {
Router::new()
}
pub fn workspaced_service() -> Router {
Router::new()
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-api/src/oidc_ee.rs
-25
View File
@@ -1,25 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2023
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
#![allow(non_snake_case)]
use axum::{routing::post, Router};
pub struct ServiceProviderExt();
pub async fn build_sp_extension() -> anyhow::Result<ServiceProviderExt> {
return Ok(ServiceProviderExt());
}
pub fn global_service() -> Router {
Router::new().route("/acs", post(acs))
}
pub async fn acs() -> String {
// Implementation is not open source as it is a Windmill Enterprise Edition feature
"SAML available only in enterprise version".to_string()
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-api/src/saml_ee.rs
-22
View File
@@ -1,22 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2023
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{middleware::Next, response::Response, routing::get, Router};
use hyper::Request;
pub fn global_service() -> Router {
Router::new().route("/ee", get(ee))
}
pub async fn ee() -> String {
return "Enterprise Edition".to_string();
}
pub async fn has_scim_token<B>(request: Request<B>, next: Next<B>) -> Response {
return next.run(request).await;
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-api/src/scim_ee.rs
-7
View File
@@ -1,7 +0,0 @@
#[cfg(feature = "stripe")]
use axum::Router;
#[cfg(feature = "stripe")]
pub fn add_stripe_routes(router: Router) -> Router {
return router;
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-api/src/stripe_ee.rs
+10
View File
@@ -3,11 +3,14 @@ use std::time::Duration;
use quick_cache::sync::Cache;
use serde::Serialize;
use tokio::{select, sync::mpsc};
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_ENABLED;
use crate::db::DB;
use crate::oauth2_ee::InstanceEvent;
#[cfg(feature = "prometheus")]
lazy_static::lazy_static! {
// TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens.
static ref WEBHOOK_REQUEST_COUNT: prometheus::Histogram = prometheus::register_histogram!(
@@ -16,6 +19,10 @@ lazy_static::lazy_static! {
)
.unwrap();
}
lazy_static::lazy_static! {
pub static ref INSTANCE_EVENTS_WEBHOOK: Option<String> = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok();
}
@@ -98,12 +105,15 @@ impl WebhookShared {
}
};
if let Some(url) = webhook_opt {
#[cfg(feature = "prometheus")]
let timer = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
let _ = client.post(url).json(&message).send().await;
#[cfg(feature = "prometheus")]
timer.map(|x| x.stop_and_record());
}
},
Some(WebhookPayload::InstanceEvent(event)) => {
#[cfg(feature = "prometheus")]
if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await;
if let Err(e) = r {
-48
View File
@@ -1,48 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::HashMap;
use windmill_common::{
error::{Error, Result},
utils::Pagination,
};
use crate::{ActionKind, AuditLog, ListAuditLogQuery};
use sqlx::{Postgres, Transaction};
#[tracing::instrument(level = "trace", skip_all)]
pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
_db: E,
_username: &str,
mut _operation: &str,
_action_kind: ActionKind,
_w_id: &str,
mut _resource: Option<&str>,
_parameters: Option<HashMap<&str, &str>>,
) -> Result<()> {
// Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
Ok(())
}
pub async fn list_audit(
_tx: Transaction<'_, Postgres>,
_w_id: String,
_pagination: Pagination,
_lq: ListAuditLogQuery,
) -> Result<Vec<AuditLog>> {
// Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
return Ok(vec![]);
}
pub async fn get_audit(tx: Transaction<'_, Postgres>, _id: i32, _w_id: &str) -> Result<AuditLog> {
// Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
tx.commit().await?;
Err(Error::NotFound(
"Audit log not not available in Windmill Community edition".to_string(),
))
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-audit/src/audit_ee.rs
-20
View File
@@ -1,20 +0,0 @@
use crate::ee::LicensePlan::Community;
use std::sync::Arc;
use tokio::sync::RwLock;
lazy_static::lazy_static! {
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
}
pub enum LicensePlan {
Community,
Pro,
Enterprise,
}
pub async fn get_license_plan() -> LicensePlan {
// Implementation is not open source
return Community;
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-common/src/ee.rs
@@ -1,17 +0,0 @@
use windmill_common::error::Result;
use crate::{DeployedObject, DB};
pub async fn handle_deployment_metadata<'c, R: rsmq_async::RsmqConnection + Send + Clone + 'c>(
_email: &str,
_created_by: &str,
_db: &DB,
_w_id: &str,
_obj: DeployedObject,
_deployment_message: Option<String>,
_rsmq: Option<R>,
_skip_db_insert: bool,
) -> Result<()> {
// Git sync is an enterprise feature and not part of the open-source version
return Ok(());
}
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-git-sync/src/git_sync_ee.rs
+1 -1
View File
@@ -10,7 +10,7 @@ path = "src/lib.rs"
[features]
default = []
prometheus = ["dep:prometheus"]
prometheus = ["dep:prometheus", "windmill-common/prometheus"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:openidconnect", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util"]
benchmark = ["windmill-queue/benchmark"]
flamegraph = []
+6 -5
View File
@@ -14,6 +14,11 @@ use prometheus::{
core::{AtomicI64, GenericGauge},
IntCounter,
};
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_ENABLED;
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_DEBUG_ENABLED;
use reqwest::Response;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sqlx::{types::Json, Pool, Postgres};
@@ -47,10 +52,6 @@ use windmill_common::{
};
#[cfg(feature = "prometheus")]
use windmill_common::worker::METRICS_ENABLED;
use windmill_queue::{
canceled_job_to_result, empty_result, get_queued_job, pull, push, CanceledBy,
PushArgs, PushIsolationLevel, WrappedError, HTTP_CLIENT,
@@ -569,8 +570,8 @@ async fn handle_receive_completed_job<
#[derive(Clone)]
pub struct JobCompletedSender(
Sender<SendResult>,
Option<Histo>,
Option<GGauge>,
Option<Histo>,
);
impl JobCompletedSender {