diff --git a/backend/src/main.rs b/backend/src/main.rs index f22473b3c0..9f2ef0d530 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -19,8 +19,8 @@ use windmill_api::HTTP_CLIENT; use windmill_common::{ global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CUSTOM_TAGS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, + DEFAULT_TAGS_PER_WORKSPACE_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, + EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, @@ -47,10 +47,10 @@ use windmill_worker::{ use crate::monitor::{ initial_load, load_keep_job_dir, load_require_preexisting_user, load_tag_per_workspace_enabled, monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting, - reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key, - reload_npm_config_registry_setting, reload_pip_index_url_setting, - reload_retention_period_setting, reload_scim_token_setting, reload_server_config, - reload_worker_config, + reload_extra_pip_index_url_setting, reload_hub_base_url_setting, + reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting, + reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, + reload_server_config, reload_worker_config, }; #[cfg(feature = "parquet")] @@ -498,7 +498,11 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not send killpill to server"); } }, - DISABLE_STATS_SETTING => {}, + HUB_BASE_URL_SETTING => { + if let Err(e) = reload_hub_base_url_setting(&db, server_mode).await { + tracing::error!(error = %e, "Could not reload hub base url setting"); + } + }, a @_ => { tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index ab514b957e..285ef3bfb1 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -14,6 +14,9 @@ use tokio::{ join, sync::{mpsc, RwLock}, }; + +#[cfg(feature = "embedding")] +use windmill_api::embeddings::update_embeddings_db; use windmill_api::{ oauth2_ee::{build_oauth_clients, OAuthClient}, DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN, @@ -24,8 +27,8 @@ use windmill_common::{ global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, + HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, }, @@ -37,7 +40,7 @@ use windmill_common::{ load_worker_config, reload_custom_tags_setting, DEFAULT_TAGS_PER_WORKSPACE, SERVER_CONFIG, WORKER_CONFIG, }, - BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED, + BASE_URL, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, METRICS_DEBUG_ENABLED, METRICS_ENABLED, }; use windmill_queue::cancel_job; use windmill_worker::{ @@ -136,6 +139,10 @@ pub async fn initial_load( tracing::error!("Error reloading base url: {:?}", e) } + if let Err(e) = reload_hub_base_url_setting(db, server_mode).await { + tracing::error!("Error reloading hub base url: {:?}", e) + } + #[cfg(feature = "parquet")] if !_is_agent { reload_s3_cache_setting(&db).await; @@ -1025,3 +1032,45 @@ async fn cancel_zombie_flow_job( ntx.commit().await?; Ok(()) } + +pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::Result<()> { + let hub_base_url = load_value_from_global_settings(db, HUB_BASE_URL_SETTING).await?; + + let base_url = if let Some(q) = hub_base_url { + if let Ok(v) = serde_json::from_value::(q.clone()) { + if v != "" { + v + } else { + DEFAULT_HUB_BASE_URL.to_string() + } + } else { + tracing::error!( + "Could not parse hub_base_url setting as a string, found: {:#?}", + &q + ); + DEFAULT_HUB_BASE_URL.to_string() + } + } else { + DEFAULT_HUB_BASE_URL.to_string() + }; + + let mut l = HUB_BASE_URL.write().await; + if server_mode { + #[cfg(feature = "embedding")] + if *l != base_url { + let disable_embedding = std::env::var("DISABLE_EMBEDDING") + .ok() + .map(|x| x.parse::().unwrap_or(false)) + .unwrap_or(false); + if !disable_embedding { + let db_clone = db.clone(); + tokio::spawn(async move { + update_embeddings_db(&db_clone).await; + }); + } + } + } + *l = base_url; + + Ok(()) +} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index e480a7ac6d..d125245d99 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -39,6 +39,7 @@ use windmill_common::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath, }, variables::build_crypt, + HUB_BASE_URL, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -633,16 +634,12 @@ async fn create_app( async fn list_hub_apps(Extension(db): Extension) -> impl IntoResponse { let (status_code, headers, body) = query_elems_from_hub( &HTTP_CLIENT, - "https://hub.windmill.dev/searchUiData?approved=true", + &format!("{}/searchUiData?approved=true", *HUB_BASE_URL.read().await), None, &db, ) .await?; - Ok::<_, Error>(( - status_code, - headers, - body - )) + Ok::<_, Error>((status_code, headers, body)) } pub async fn get_hub_app_by_id( @@ -651,7 +648,7 @@ pub async fn get_hub_app_by_id( ) -> JsonResult { let value = http_get_from_hub( &HTTP_CLIENT, - &format!("https://hub.windmill.dev/apps/{id}/json"), + &format!("{}/apps/{}/json", *HUB_BASE_URL.read().await, id), false, None, &db, diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index a668e0f716..fc9d36713f 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -1,21 +1,22 @@ #[cfg(feature = "embedding")] +use anyhow::{anyhow, Error, Result}; +#[cfg(feature = "embedding")] use std::{collections::HashMap, path::PathBuf, sync::Arc}; #[cfg(feature = "embedding")] -use anyhow::{anyhow, Error, Result}; +use windmill_common::DEFAULT_HUB_BASE_URL; +#[cfg(feature = "embedding")] +use windmill_common::HUB_BASE_URL; use axum::Router; #[cfg(feature = "embedding")] use axum::{ extract::{Path, Query}, - Json, + Json, }; - #[cfg(feature = "embedding")] -use axum::{ - routing::get, Extension -}; +use axum::routing::get; #[cfg(feature = "embedding")] use candle_core::{Device, Tensor}; #[cfg(feature = "embedding")] @@ -45,9 +46,14 @@ use windmill_common::utils::http_get_from_hub; use windmill_common::error::JsonResult; #[cfg(feature = "embedding")] - use crate::{resources::ResourceType, HTTP_CLIENT}; +#[cfg(feature = "embedding")] +lazy_static::lazy_static! { + pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); + pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); +} + #[cfg(feature = "embedding")] #[derive(Deserialize)] struct HubScriptsQuery { @@ -71,9 +77,8 @@ pub struct HubScriptResult { #[cfg(feature = "embedding")] async fn query_hub_scripts( Query(query): Query, - Extension(embeddings_db): Extension>>>, ) -> JsonResult> { - let embeddings_db = embeddings_db.read().await; + let embeddings_db = EMBEDDINGS_DB.read().await; if let Some(embeddings_db) = embeddings_db.as_ref() { let results = embeddings_db @@ -88,7 +93,6 @@ async fn query_hub_scripts( } } - #[cfg(feature = "embedding")] #[derive(Deserialize)] struct ResourceTypesQuery { @@ -106,9 +110,8 @@ pub struct ResourceTypeResult { async fn query_resource_types( Query(query): Query, Path(w_id): Path, - Extension(embeddings_db): Extension>>>, ) -> JsonResult> { - let embeddings_db = embeddings_db.read().await; + let embeddings_db = EMBEDDINGS_DB.read().await; if let Some(embeddings_db) = embeddings_db.as_ref() { let results = embeddings_db @@ -123,7 +126,6 @@ async fn query_resource_types( } } - #[cfg(feature = "embedding")] #[derive(Deserialize, Debug, Clone)] struct HubScript { @@ -292,24 +294,41 @@ impl EmbeddingsDb { self.db .create_collection("resource_types".to_string(), 384, Distance::Cosine)?; - let response = HTTP_CLIENT - .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json") - .send() - .await; - let response = - if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { - tracing::warn!("Failed to get scripts embeddings from bucket, trying hub..."); + let hub_base_url = HUB_BASE_URL.read().await.clone(); + + let response = match hub_base_url.as_str() { + DEFAULT_HUB_BASE_URL => { + let response = HTTP_CLIENT + .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json") + .send() + .await; + + if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { + tracing::warn!("Failed to get scripts embeddings from bucket, trying hub..."); + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/embeddings", hub_base_url), + false, + None, + pg_db, + ) + .await? + } else { + response.unwrap() + } + } + _ => { http_get_from_hub( &HTTP_CLIENT, - "https://hub.windmill.dev/scripts/embeddings", + &format!("{}/scripts/embeddings", hub_base_url), false, None, pg_db, ) .await? - } else { - response.unwrap() - }; + } + }; + if response.error_for_status_ref().is_err() { return Err(anyhow!( "Failed to get scripts embeddings from hub with error code: {}", @@ -338,25 +357,40 @@ impl EmbeddingsDb { self.db.insert_into_collection("scripts", embedding)?; } - let response = HTTP_CLIENT - .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json") - .send() - .await; - let response = if response.is_err() - || response.as_ref().unwrap().error_for_status_ref().is_err() - { - tracing::warn!("Failed to get resource types embeddings from bucket, trying hub..."); - http_get_from_hub( - &HTTP_CLIENT, - "https://hub.windmill.dev/resource_types/embeddings", - false, - None, - pg_db, - ) - .await? - } else { - response.unwrap() + let response = match hub_base_url.as_str() { + DEFAULT_HUB_BASE_URL => { + let response = HTTP_CLIENT + .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json") + .send() + .await; + if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { + tracing::warn!( + "Failed to get resource types embeddings from bucket, trying hub..." + ); + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/resource_types/embeddings", hub_base_url), + false, + None, + pg_db, + ) + .await? + } else { + response.unwrap() + } + } + _ => { + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/resource_types/embeddings", hub_base_url), + false, + None, + pg_db, + ) + .await? + } }; + if response.error_for_status_ref().is_err() { return Err(anyhow!( "Failed to get resource types embeddings from hub with error code: {}", @@ -554,9 +588,7 @@ fn normalize_l2(v: &Tensor) -> Result { } #[cfg(feature = "embedding")] -pub fn load_embeddings_db(db: &Pool) -> Arc>> { - let embeddings_db: Arc>> = Arc::new(RwLock::new(None)); - +pub fn load_embeddings_db(db: &Pool) -> () { let disable_embedding = std::env::var("DISABLE_EMBEDDING") .ok() .map(|x| x.parse::().unwrap_or(false)) @@ -564,24 +596,14 @@ pub fn load_embeddings_db(db: &Pool) -> Arc>> = embeddings_db.clone(); tokio::spawn(async move { let model_instance = ModelInstance::new().await; - if let Ok(model_instance) = model_instance { - let model_instance = Arc::new(model_instance); + let mut model_instance_lock = MODEL_INSTANCE.write().await; + *model_instance_lock = Some(Arc::new(model_instance)); + drop(model_instance_lock); loop { - tracing::info!("Creating embeddings DB..."); - let new_embeddings_db = - EmbeddingsDb::new(&db_clone, model_instance.clone()).await; - if let Err(e) = new_embeddings_db.as_ref() { - tracing::error!("Failed to create embeddings db: {}", e); - } else { - let mut embeddings_db = embeddings_clone.write().await; - *embeddings_db = new_embeddings_db.ok(); - tracing::info!("Created embeddings DB"); - } - + update_embeddings_db(&db_clone).await; tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await; } } else { @@ -592,39 +614,41 @@ pub fn load_embeddings_db(db: &Pool) -> Arc>>>) -> Router { - if let Some(embeddings_db) = embeddings_db { - Router::new() - .route("/query_resource_types", get(query_resource_types)) - .layer(Extension(embeddings_db)) +pub async fn update_embeddings_db(db: &Pool) -> () { + if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() { + tracing::info!("Creating embeddings DB..."); + let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await; + if let Err(e) = new_embeddings_db.as_ref() { + tracing::error!("Failed to create embeddings db: {}", e); + } else { + let mut embeddings_db = EMBEDDINGS_DB.write().await; + *embeddings_db = new_embeddings_db.ok(); + tracing::info!("Created embeddings DB"); + } } else { - Router::new() + tracing::error!("Could not update embeddings DB, model instance not initialized"); } } #[cfg(feature = "embedding")] -pub fn global_service(embeddings_db: Option>>>) -> Router { - if let Some(embeddings_db) = embeddings_db { - Router::new() - .route("/query_hub_scripts", get(query_hub_scripts)) - .layer(Extension(embeddings_db)) - } else { - Router::new() - } +pub fn workspaced_service() -> Router { + Router::new().route("/query_resource_types", get(query_resource_types)) } - -#[cfg(not(feature = "embedding"))] -pub fn workspaced_service(_embeddings_db: Option<()>) -> Router { - Router::new() +#[cfg(feature = "embedding")] +pub fn global_service() -> Router { + Router::new().route("/query_hub_scripts", get(query_hub_scripts)) } #[cfg(not(feature = "embedding"))] -pub fn global_service(_embeddings_db: Option<()>) -> Router { - Router::new() +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "embedding"))] +pub fn global_service() -> Router { + Router::new() } diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 44c71b7498..e0fbdd4437 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -31,6 +31,7 @@ use sqlx::{FromRow, Postgres, Transaction}; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; use windmill_common::utils::query_elems_from_hub; +use windmill_common::HUB_BASE_URL; use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error, JsonResult, Result}, @@ -167,7 +168,10 @@ async fn list_flows( async fn list_hub_flows(Extension(db): Extension) -> impl IntoResponse { let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, - "https://hub.windmill.dev/searchFlowData?approved=true", + &format!( + "{}/searchFlowData?approved=true", + *HUB_BASE_URL.read().await + ), None, &db, ) @@ -199,7 +203,7 @@ pub async fn get_hub_flow_by_id( ) -> JsonResult { let value = http_get_from_hub( &HTTP_CLIENT, - &format!("https://hub.windmill.dev/flows/{id}/json"), + &format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id), false, None, &db, diff --git a/backend/windmill-api/src/integration.rs b/backend/windmill-api/src/integration.rs index 657c3019c7..07a719b337 100644 --- a/backend/windmill-api/src/integration.rs +++ b/backend/windmill-api/src/integration.rs @@ -1,8 +1,6 @@ use crate::{db::DB, HTTP_CLIENT}; -use axum::{ - extract::Query, response::IntoResponse, routing::get, Extension, Router, -}; -use windmill_common::{error::Error, utils::query_elems_from_hub}; +use axum::{extract::Query, response::IntoResponse, routing::get, Extension, Router}; +use windmill_common::{error::Error, utils::query_elems_from_hub, HUB_BASE_URL}; pub fn global_service() -> Router { Router::new().route("/hub/list", get(list_hub_integrations)) @@ -24,14 +22,10 @@ async fn list_hub_integrations( let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, - "https://hub.windmill.dev/integrations/list", + &format!("{}/integrations/list", *HUB_BASE_URL.read().await), Some(query_params), &db, ) .await?; - Ok::<_, Error>(( - status_code, - headers, - response - )) + Ok::<_, Error>((status_code, headers, response)) } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 8604464fb8..c107a514ac 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -171,18 +171,10 @@ pub async fn run_server( let sp_extension = Arc::new(saml_ee::build_sp_extension().await?); - let embeddings_db = if server_mode { + if server_mode { #[cfg(feature = "embedding")] - { - Some(load_embeddings_db(&db)) - } - #[cfg(not(feature = "embedding"))] - { - Some(()) - } - } else { - None - }; + load_embeddings_db(&db) + } let job_helpers_service = { #[cfg(feature = "parquet")] @@ -209,10 +201,7 @@ pub async fn run_server( .nest("/apps", apps::workspaced_service()) .nest("/audit", audit::workspaced_service()) .nest("/capture", capture::workspaced_service()) - .nest( - "/embeddings", - embeddings::workspaced_service(embeddings_db.clone()), - ) + .nest("/embeddings", embeddings::workspaced_service()) .nest("/drafts", drafts::workspaced_service()) .nest("/favorites", favorite::workspaced_service()) .nest("/flows", flows::workspaced_service()) @@ -250,10 +239,7 @@ pub async fn run_server( .nest("/flows", flows::global_service()) .nest("/apps", apps::global_service().layer(cors.clone())) .nest("/schedules", schedule::global_service()) - .nest( - "/embeddings", - embeddings::global_service(embeddings_db.clone()), - ) + .nest("/embeddings", embeddings::global_service()) .route_layer(from_extractor::()) .route_layer(from_extractor::()) .nest("/jobs", jobs::global_root_service()) diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index c96f4c1338..a86116647b 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -44,6 +44,7 @@ use windmill_common::{ utils::{ not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath, }, + HUB_BASE_URL, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use windmill_parser_ts::remove_pinned_imports; @@ -290,7 +291,7 @@ async fn get_top_hub_scripts( let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, - "https://hub.windmill.dev/scripts/top", + &format!("{}/scripts/top", *HUB_BASE_URL.read().await), Some(query_params), &db, ) diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 197b529014..7bd13ccfab 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -26,7 +26,7 @@ use serde::Deserialize; use tokio::time::timeout; use windmill_common::{ error::{self, to_anyhow, JsonResult, Result}, - global_settings::{AUTOMATE_USERNAME_CREATION_SETTING, ENV_SETTINGS}, + global_settings::{AUTOMATE_USERNAME_CREATION_SETTING, ENV_SETTINGS, HUB_BASE_URL_SETTING}, server::Smtp, }; @@ -147,7 +147,6 @@ pub async fn test_s3_bucket( Ok("Tested blob storage successfully".to_string()) } - #[derive(Deserialize)] pub struct TestKey { pub license_key: String, @@ -251,6 +250,7 @@ pub async fn get_global_setting( if !key.starts_with("default_error_handler_") && !key.starts_with("default_recovery_handler_") && key != AUTOMATE_USERNAME_CREATION_SETTING + && key != HUB_BASE_URL_SETTING { require_super_admin(&db, &authed.email).await?; } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 50556a73b1..d24b6b7070 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -23,6 +23,7 @@ pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexistin pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; +pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const ENV_SETTINGS: [&str; 50] = [ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 72cfe86f6e..c90f0e702c 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -46,6 +46,8 @@ pub mod tracing_init; pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50; pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 4; +pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; + lazy_static::lazy_static! { pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT") .ok() @@ -69,6 +71,8 @@ lazy_static::lazy_static! { pub static ref BASE_URL: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + + pub static ref HUB_BASE_URL: Arc> = Arc::new(RwLock::new(DEFAULT_HUB_BASE_URL.to_string())); } pub async fn shutdown_signal( diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 05aa5ca032..61c7f3e8cc 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -14,7 +14,7 @@ use std::{ use crate::{ error::{to_anyhow, Error}, utils::http_get_from_hub, - DB, + DB, HUB_BASE_URL, }; use serde::de::Error as _; use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; @@ -353,7 +353,7 @@ pub async fn get_hub_script_by_path( let content = http_get_from_hub( http_client, - &format!("https://hub.windmill.dev/raw/{path}.ts"), + &format!("{}/raw/{}.ts", *HUB_BASE_URL.read().await, path), true, None, db, @@ -377,7 +377,7 @@ pub async fn get_full_hub_script_by_path( let value = http_get_from_hub( http_client, - &format!("https://hub.windmill.dev/raw2/{path}"), + &format!("{}/raw2/{}", *HUB_BASE_URL.read().await, path), true, None, db, diff --git a/cli/hub.ts b/cli/hub.ts index da0abbbd12..751a412410 100644 --- a/cli/hub.ts +++ b/cli/hub.ts @@ -21,6 +21,11 @@ async function pull(opts: GlobalOptions) { key: "uid", }); + const hubBaseUrl = + (await SettingService.getGlobal({ + key: "hubBaseUrl", + })) ?? "https://hub.windmill.dev"; + const headers = { Accept: "application/json", "X-email": userInfo.email, @@ -40,20 +45,17 @@ async function pull(opts: GlobalOptions) { created_by: string; created_at: Date; comments: never[]; - }[] = await fetch("https://hub.windmill.dev/resource_types/list", { + }[] = await fetch(hubBaseUrl + "/resource_types/list", { headers, }) .then((r) => r.json()) .then((list: { id: number; name: string }[]) => list.map((x) => - fetch( - "https://hub.windmill.dev/resource_types/" + x.id + "/" + x.name, - { - headers: { - Accept: "application/json", - }, - } - ) + fetch(hubBaseUrl + "/resource_types/" + x.id + "/" + x.name, { + headers: { + Accept: "application/json", + }, + }) ) ) .then((x) => Promise.all(x)) diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index ff061aa0e7..8e2b3ef696 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -15,6 +15,7 @@ type Flow } from '$lib/gen' import { inferArgs } from '$lib/infer' + import { hubBaseUrlStore } from '$lib/stores' import { CheckCircle2, Loader2, RotateCw, XCircle } from 'lucide-svelte' @@ -221,7 +222,7 @@
Example of error handler scripts can be found on Windmill Hub diff --git a/frontend/src/lib/components/FlowGraphViewerStep.svelte b/frontend/src/lib/components/FlowGraphViewerStep.svelte index b4c25974a3..21675d3eb6 100644 --- a/frontend/src/lib/components/FlowGraphViewerStep.svelte +++ b/frontend/src/lib/components/FlowGraphViewerStep.svelte @@ -11,6 +11,7 @@ import SchemaViewer from './SchemaViewer.svelte' import { scriptPathToHref } from '$lib/scripts' import { cleanExpr } from '$lib/utils' + import { hubBaseUrlStore } from '$lib/stores' import { twMerge } from 'tailwind-merge' import FlowModuleScript from './flows/content/FlowModuleScript.svelte' @@ -34,7 +35,7 @@ @@ -52,7 +53,7 @@ class="w-full h-full text-sm" title="embedded script from hub" frameborder="0" - src="https://hub.windmill.dev/embed/script/{stepDetail.value?.path?.substring(4)}" + src="{$hubBaseUrlStore}/embed/script/{stepDetail.value?.path?.substring(4)}" />
{/if} @@ -116,7 +117,7 @@ @@ -166,7 +167,7 @@ class="w-full grow text-sm" title="embedded script from hub" frameborder="0" - src="https://hub.windmill.dev/embed/script/{stepDetail.value?.path?.substring(4)}" + src="{$hubBaseUrlStore}/embed/script/{stepDetail.value?.path?.substring(4)}" /> {:else} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index 91dda224cc..425b864f06 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -628,7 +628,7 @@ // displayName: 'Publish to Hub', // icon: faGlobe, // action: () => { - // const url = appToHubUrl(toStatic($app, $staticExporter, $summary)) + // const url = appToHubUrl(toStatic($app, $staticExporter, $summary, $hubBaseUrlStore)) // window.open(url.toString(), '_blank') // } // }, diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index d083c0a271..ea4e2c3fea 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -110,6 +110,14 @@ export const settings: Record = { fieldType: 'text', storage: 'setting', ee_only: '' + }, + { + label: 'Private hub base url', + description: 'Base url of your private hub instance', + key: 'hub_base_url', + fieldType: 'text', + storage: 'setting', + ee_only: '' } ], 'SSO/OAuth': [], diff --git a/frontend/src/lib/hub.ts b/frontend/src/lib/hub.ts index 0aa2c79781..2b1a6ddff5 100644 --- a/frontend/src/lib/hub.ts +++ b/frontend/src/lib/hub.ts @@ -9,9 +9,10 @@ export function scriptToHubUrl( kind: Script.kind, language: Script.language, schema: Schema | any, - lock: string | undefined + lock: string | undefined, + hubBaseUrl: string ): URL { - const url = new URL('https://hub.windmill.dev/scripts/add') + const url = new URL(hubBaseUrl + '/scripts/add') url.hash = encodeState({ content, summary, description, kind, language, schema, lock }) return url @@ -37,8 +38,8 @@ export async function loadHubApps() { } } -export function flowToHubUrl(flow: Flow): URL { - const url = new URL('https://hub.windmill.dev/flows/add') +export function flowToHubUrl(flow: Flow, hubBaseUrl: string): URL { + const url = new URL(hubBaseUrl + '/flows/add') const openFlow = { value: flow.value, summary: flow.summary, @@ -49,8 +50,8 @@ export function flowToHubUrl(flow: Flow): URL { return url } -export function appToHubUrl(staticApp: any): URL { - const url = new URL('https://hub.windmill.dev/apps/add') +export function appToHubUrl(staticApp: any, hubBaseUrl: string): URL { + const url = new URL(hubBaseUrl + '/apps/add') url.searchParams.append('app', encodeState(staticApp)) return url } diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 89639ca60c..bcd0dca354 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -79,9 +79,9 @@ export async function loadSchemaFlow(path: string): Promise { return flow.schema as any } -export function scriptPathToHref(path: string): string { +export function scriptPathToHref(path: string, hubBaseUrl: string): string { if (path.startsWith('hub/')) { - return 'https://hub.windmill.dev/from_version/' + path.substring(4) + return hubBaseUrl + '/from_version/' + path.substring(4) } else { return `/scripts/get/${path}?workspace=${get(workspaceStore)}` } diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 1da548cd45..76930edca8 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -47,6 +47,7 @@ export const starStore = writable(1) export const usersWorkspaceStore = writable(undefined) export const superadmin = writable(undefined) export const lspTokenStore = writable(undefined) +export const hubBaseUrlStore = writable('https://hub.windmill.dev') export const userWorkspaces: Readable< Array<{ id: string diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 303a7303ca..15303fb61c 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -7,6 +7,7 @@ OpenAPI, RawAppService, ScriptService, + SettingService, UserService, WorkspaceService } from '$lib/gen' @@ -23,7 +24,8 @@ userStore, workspaceStore, type UserExt, - defaultScripts + defaultScripts, + hubBaseUrlStore } from '$lib/stores' import CenteredModal from '$lib/components/CenteredModal.svelte' import { afterNavigate, beforeNavigate, goto } from '$app/navigation' @@ -113,6 +115,7 @@ loadFavorites() loadUsage() syncTutorialsTodos() + loadHubBaseUrl() } async function loadUsage() { @@ -124,6 +127,11 @@ } } + async function loadHubBaseUrl() { + $hubBaseUrlStore = + (await SettingService.getGlobal({ key: 'hub_base_url' })) ?? 'https://hub.windmill.dev' + } + async function loadFavorites() { const scripts = await ScriptService.listScripts({ workspace: $workspaceStore ?? '', diff --git a/frontend/src/routes/(root)/(logged)/+page.svelte b/frontend/src/routes/(root)/(logged)/+page.svelte index e5e1123df0..e889f5b82b 100644 --- a/frontend/src/routes/(root)/(logged)/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/+page.svelte @@ -12,6 +12,7 @@ import FlowViewer from '$lib/components/FlowViewer.svelte' import HighlightCode from '$lib/components/HighlightCode.svelte' import { Building, ExternalLink, GitFork, Globe2, Loader2 } from 'lucide-svelte' + import { hubBaseUrlStore } from '$lib/stores' import ItemsList from '$lib/components/home/ItemsList.svelte' import CreateActionsApp from '$lib/components/flows/CreateActionsApp.svelte' @@ -82,8 +83,7 @@ diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 3bb29f440c..2b1069c3ce 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -10,7 +10,7 @@ } from '$lib/utils' import Tooltip from '$lib/components/Tooltip.svelte' import ShareModal from '$lib/components/ShareModal.svelte' - import { runFormStore, userStore, workspaceStore } from '$lib/stores' + import { hubBaseUrlStore, runFormStore, userStore, workspaceStore } from '$lib/stores' import SchemaViewer from '$lib/components/SchemaViewer.svelte' import { onDestroy } from 'svelte' import HighlightCode from '$lib/components/HighlightCode.svelte' @@ -374,7 +374,8 @@ script.kind, script.language, script.schema, - script.lock ?? '' + script.lock ?? '', + $hubBaseUrlStore ).toString(), '_blank' ) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index a57e1a1bd3..3b9b0b548a 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -29,7 +29,8 @@ superadmin, userStore, usersWorkspaceStore, - workspaceStore + workspaceStore, + hubBaseUrlStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { setQueryWithoutLoad, emptyString, tryEvery } from '$lib/utils' @@ -799,10 +800,8 @@
Pick a script or flow meant to be triggered when the `/windmill` command is invoked. Upon - connection, templates for a script - and flow are available. + connection, templates for a script + and flow are available.