starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-11 00:06:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: private hub (#3491)

Browse Source 
* feat: private hub v0

* feat: add UI setting

* Update ee.rs

* Update ee.rs

* fix: remove ee symlink

* fix: add back CE ee.rs

* feat: disable s3 embedding loading when private hub

* feat: reload embeddings on hub url change

* fix: only reload embeddings db on server mode

* patch: set default hub url const

* fix: nit
This commit is contained in:
HugoCasa
2024-04-04 04:16:53 +02:00
committed by GitHub
parent 92e8b07c17
commit 1d7bab075b
24 changed files with 259 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/main.rs
+11 -7
View File
@@ -19,8 +19,8 @@ use windmill_api::HTTP_CLIENT;
use windmill_common::{
global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CUSTOM_TAGS_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
@@ -47,10 +47,10 @@ use windmill_worker::{
use crate::monitor::{
initial_load, load_keep_job_dir, load_require_preexisting_user, load_tag_per_workspace_enabled,
monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting,
reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key,
reload_npm_config_registry_setting, reload_pip_index_url_setting,
reload_retention_period_setting, reload_scim_token_setting, reload_server_config,
reload_worker_config,
reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
reload_job_default_timeout_setting, reload_license_key, reload_npm_config_registry_setting,
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
reload_server_config, reload_worker_config,
};
#[cfg(feature = "parquet")]
@@ -498,7 +498,11 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!(error = %e, "Could not send killpill to server");
}
},
DISABLE_STATS_SETTING => {},
HUB_BASE_URL_SETTING => {
if let Err(e) = reload_hub_base_url_setting(&db, server_mode).await {
tracing::error!(error = %e, "Could not reload hub base url setting");
}
},
a @_ => {
tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
}
backend/src/monitor.rs
+52 -3
View File
@@ -14,6 +14,9 @@ use tokio::{
join,
sync::{mpsc, RwLock},
};
#[cfg(feature = "embedding")]
use windmill_api::embeddings::update_embeddings_db;
use windmill_api::{
oauth2_ee::{build_oauth_clients, OAuthClient},
DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
@@ -24,8 +27,8 @@ use windmill_common::{
global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
},
@@ -37,7 +40,7 @@ use windmill_common::{
load_worker_config, reload_custom_tags_setting, DEFAULT_TAGS_PER_WORKSPACE, SERVER_CONFIG,
WORKER_CONFIG,
},
BASE_URL, DB, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
BASE_URL, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -136,6 +139,10 @@ pub async fn initial_load(
tracing::error!("Error reloading base url: {:?}", e)
}
if let Err(e) = reload_hub_base_url_setting(db, server_mode).await {
tracing::error!("Error reloading hub base url: {:?}", e)
}
#[cfg(feature = "parquet")]
if !_is_agent {
reload_s3_cache_setting(&db).await;
@@ -1025,3 +1032,45 @@ async fn cancel_zombie_flow_job(
ntx.commit().await?;
Ok(())
}
pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::Result<()> {
let hub_base_url = load_value_from_global_settings(db, HUB_BASE_URL_SETTING).await?;
let base_url = if let Some(q) = hub_base_url {
if let Ok(v) = serde_json::from_value::<String>(q.clone()) {
if v != "" {
v
} else {
DEFAULT_HUB_BASE_URL.to_string()
}
} else {
tracing::error!(
"Could not parse hub_base_url setting as a string, found: {:#?}",
&q
);
DEFAULT_HUB_BASE_URL.to_string()
}
} else {
DEFAULT_HUB_BASE_URL.to_string()
};
let mut l = HUB_BASE_URL.write().await;
if server_mode {
#[cfg(feature = "embedding")]
if *l != base_url {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false);
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
update_embeddings_db(&db_clone).await;
});
}
}
}
*l = base_url;
Ok(())
}
backend/windmill-api/src/apps.rs
+4 -7
View File
@@ -39,6 +39,7 @@ use windmill_common::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
},
variables::build_crypt,
HUB_BASE_URL,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -633,16 +634,12 @@ async fn create_app(
async fn list_hub_apps(Extension(db): Extension<DB>) -> impl IntoResponse {
let (status_code, headers, body) = query_elems_from_hub(
&HTTP_CLIENT,
"https://hub.windmill.dev/searchUiData?approved=true",
&format!("{}/searchUiData?approved=true", *HUB_BASE_URL.read().await),
None,
&db,
)
.await?;
Ok::<_, Error>((
status_code,
headers,
body
))
Ok::<_, Error>((status_code, headers, body))
}
pub async fn get_hub_app_by_id(
@@ -651,7 +648,7 @@ pub async fn get_hub_app_by_id(
) -> JsonResult<serde_json::Value> {
let value = http_get_from_hub(
&HTTP_CLIENT,
&format!("https://hub.windmill.dev/apps/{id}/json"),
&format!("{}/apps/{}/json", *HUB_BASE_URL.read().await, id),
false,
None,
&db,
backend/windmill-api/src/embeddings.rs
+105 -81
View File
@@ -1,21 +1,22 @@
#[cfg(feature = "embedding")]
use anyhow::{anyhow, Error, Result};
#[cfg(feature = "embedding")]
use std::{collections::HashMap, path::PathBuf, sync::Arc};
#[cfg(feature = "embedding")]
use anyhow::{anyhow, Error, Result};
use windmill_common::DEFAULT_HUB_BASE_URL;
#[cfg(feature = "embedding")]
use windmill_common::HUB_BASE_URL;
use axum::Router;
#[cfg(feature = "embedding")]
use axum::{
extract::{Path, Query},
Json,
Json,
};
#[cfg(feature = "embedding")]
use axum::{
routing::get, Extension
};
use axum::routing::get;
#[cfg(feature = "embedding")]
use candle_core::{Device, Tensor};
#[cfg(feature = "embedding")]
@@ -45,9 +46,14 @@ use windmill_common::utils::http_get_from_hub;
use windmill_common::error::JsonResult;
#[cfg(feature = "embedding")]
use crate::{resources::ResourceType, HTTP_CLIENT};
#[cfg(feature = "embedding")]
lazy_static::lazy_static! {
pub static ref EMBEDDINGS_DB: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
pub static ref MODEL_INSTANCE: Arc<RwLock<Option<Arc<ModelInstance>>>> = Arc::new(RwLock::new(None));
}
#[cfg(feature = "embedding")]
#[derive(Deserialize)]
struct HubScriptsQuery {
@@ -71,9 +77,8 @@ pub struct HubScriptResult {
#[cfg(feature = "embedding")]
async fn query_hub_scripts(
Query(query): Query<HubScriptsQuery>,
Extension(embeddings_db): Extension<Arc<RwLock<Option<EmbeddingsDb>>>>,
) -> JsonResult<Vec<HubScriptResult>> {
let embeddings_db = embeddings_db.read().await;
let embeddings_db = EMBEDDINGS_DB.read().await;
if let Some(embeddings_db) = embeddings_db.as_ref() {
let results = embeddings_db
@@ -88,7 +93,6 @@ async fn query_hub_scripts(
}
}
#[cfg(feature = "embedding")]
#[derive(Deserialize)]
struct ResourceTypesQuery {
@@ -106,9 +110,8 @@ pub struct ResourceTypeResult {
async fn query_resource_types(
Query(query): Query<ResourceTypesQuery>,
Path(w_id): Path<String>,
Extension(embeddings_db): Extension<Arc<RwLock<Option<EmbeddingsDb>>>>,
) -> JsonResult<Vec<ResourceTypeResult>> {
let embeddings_db = embeddings_db.read().await;
let embeddings_db = EMBEDDINGS_DB.read().await;
if let Some(embeddings_db) = embeddings_db.as_ref() {
let results = embeddings_db
@@ -123,7 +126,6 @@ async fn query_resource_types(
}
}
#[cfg(feature = "embedding")]
#[derive(Deserialize, Debug, Clone)]
struct HubScript {
@@ -292,24 +294,41 @@ impl EmbeddingsDb {
self.db
.create_collection("resource_types".to_string(), 384, Distance::Cosine)?;
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json")
.send()
.await;
let response =
if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
tracing::warn!("Failed to get scripts embeddings from bucket, trying hub...");
let hub_base_url = HUB_BASE_URL.read().await.clone();
let response = match hub_base_url.as_str() {
DEFAULT_HUB_BASE_URL => {
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json")
.send()
.await;
if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
tracing::warn!("Failed to get scripts embeddings from bucket, trying hub...");
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
pg_db,
)
.await?
} else {
response.unwrap()
}
}
_ => {
http_get_from_hub(
&HTTP_CLIENT,
"https://hub.windmill.dev/scripts/embeddings",
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
pg_db,
)
.await?
} else {
response.unwrap()
};
}
};
if response.error_for_status_ref().is_err() {
return Err(anyhow!(
"Failed to get scripts embeddings from hub with error code: {}",
@@ -338,25 +357,40 @@ impl EmbeddingsDb {
self.db.insert_into_collection("scripts", embedding)?;
}
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json")
.send()
.await;
let response = if response.is_err()
|| response.as_ref().unwrap().error_for_status_ref().is_err()
{
tracing::warn!("Failed to get resource types embeddings from bucket, trying hub...");
http_get_from_hub(
&HTTP_CLIENT,
"https://hub.windmill.dev/resource_types/embeddings",
false,
None,
pg_db,
)
.await?
} else {
response.unwrap()
let response = match hub_base_url.as_str() {
DEFAULT_HUB_BASE_URL => {
let response = HTTP_CLIENT
.get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json")
.send()
.await;
if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() {
tracing::warn!(
"Failed to get resource types embeddings from bucket, trying hub..."
);
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
pg_db,
)
.await?
} else {
response.unwrap()
}
}
_ => {
http_get_from_hub(
&HTTP_CLIENT,
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
pg_db,
)
.await?
}
};
if response.error_for_status_ref().is_err() {
return Err(anyhow!(
"Failed to get resource types embeddings from hub with error code: {}",
@@ -554,9 +588,7 @@ fn normalize_l2(v: &Tensor) -> Result<Tensor> {
}
#[cfg(feature = "embedding")]
pub fn load_embeddings_db(db: &Pool<Postgres>) -> Arc<RwLock<Option<EmbeddingsDb>>> {
let embeddings_db: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
pub fn load_embeddings_db(db: &Pool<Postgres>) -> () {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
@@ -564,24 +596,14 @@ pub fn load_embeddings_db(db: &Pool<Postgres>) -> Arc<RwLock<Option<EmbeddingsDb
if !disable_embedding {
let db_clone = db.clone();
let embeddings_clone: Arc<RwLock<Option<EmbeddingsDb>>> = embeddings_db.clone();
tokio::spawn(async move {
let model_instance = ModelInstance::new().await;
if let Ok(model_instance) = model_instance {
let model_instance = Arc::new(model_instance);
let mut model_instance_lock = MODEL_INSTANCE.write().await;
*model_instance_lock = Some(Arc::new(model_instance));
drop(model_instance_lock);
loop {
tracing::info!("Creating embeddings DB...");
let new_embeddings_db =
EmbeddingsDb::new(&db_clone, model_instance.clone()).await;
if let Err(e) = new_embeddings_db.as_ref() {
tracing::error!("Failed to create embeddings db: {}", e);
} else {
let mut embeddings_db = embeddings_clone.write().await;
*embeddings_db = new_embeddings_db.ok();
tracing::info!("Created embeddings DB");
}
update_embeddings_db(&db_clone).await;
tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await;
}
} else {
@@ -592,39 +614,41 @@ pub fn load_embeddings_db(db: &Pool<Postgres>) -> Arc<RwLock<Option<EmbeddingsDb
}
});
}
embeddings_db
}
#[cfg(feature = "embedding")]
pub fn workspaced_service(embeddings_db: Option<Arc<RwLock<Option<EmbeddingsDb>>>>) -> Router {
if let Some(embeddings_db) = embeddings_db {
Router::new()
.route("/query_resource_types", get(query_resource_types))
.layer(Extension(embeddings_db))
pub async fn update_embeddings_db(db: &Pool<Postgres>) -> () {
if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() {
tracing::info!("Creating embeddings DB...");
let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await;
if let Err(e) = new_embeddings_db.as_ref() {
tracing::error!("Failed to create embeddings db: {}", e);
} else {
let mut embeddings_db = EMBEDDINGS_DB.write().await;
*embeddings_db = new_embeddings_db.ok();
tracing::info!("Created embeddings DB");
}
} else {
Router::new()
tracing::error!("Could not update embeddings DB, model instance not initialized");
}
}
#[cfg(feature = "embedding")]
pub fn global_service(embeddings_db: Option<Arc<RwLock<Option<EmbeddingsDb>>>>) -> Router {
if let Some(embeddings_db) = embeddings_db {
Router::new()
.route("/query_hub_scripts", get(query_hub_scripts))
.layer(Extension(embeddings_db))
} else {
Router::new()
}
pub fn workspaced_service() -> Router {
Router::new().route("/query_resource_types", get(query_resource_types))
}
#[cfg(not(feature = "embedding"))]
pub fn workspaced_service(_embeddings_db: Option<()>) -> Router {
Router::new()
#[cfg(feature = "embedding")]
pub fn global_service() -> Router {
Router::new().route("/query_hub_scripts", get(query_hub_scripts))
}
#[cfg(not(feature = "embedding"))]
pub fn global_service(_embeddings_db: Option<()>) -> Router {
Router::new()
pub fn workspaced_service() -> Router {
Router::new()
}
#[cfg(not(feature = "embedding"))]
pub fn global_service() -> Router {
Router::new()
}
backend/windmill-api/src/flows.rs
+6 -2
View File
@@ -31,6 +31,7 @@ use sqlx::{FromRow, Postgres, Transaction};
use windmill_audit::audit_ee::audit_log;
use windmill_audit::ActionKind;
use windmill_common::utils::query_elems_from_hub;
use windmill_common::HUB_BASE_URL;
use windmill_common::{
db::UserDB,
error::{self, to_anyhow, Error, JsonResult, Result},
@@ -167,7 +168,10 @@ async fn list_flows(
async fn list_hub_flows(Extension(db): Extension<DB>) -> impl IntoResponse {
let (status_code, headers, response) = query_elems_from_hub(
&HTTP_CLIENT,
"https://hub.windmill.dev/searchFlowData?approved=true",
&format!(
"{}/searchFlowData?approved=true",
*HUB_BASE_URL.read().await
),
None,
&db,
)
@@ -199,7 +203,7 @@ pub async fn get_hub_flow_by_id(
) -> JsonResult<serde_json::Value> {
let value = http_get_from_hub(
&HTTP_CLIENT,
&format!("https://hub.windmill.dev/flows/{id}/json"),
&format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id),
false,
None,
&db,
backend/windmill-api/src/integration.rs
+4 -10
View File
@@ -1,8 +1,6 @@
use crate::{db::DB, HTTP_CLIENT};
use axum::{
extract::Query, response::IntoResponse, routing::get, Extension, Router,
};
use windmill_common::{error::Error, utils::query_elems_from_hub};
use axum::{extract::Query, response::IntoResponse, routing::get, Extension, Router};
use windmill_common::{error::Error, utils::query_elems_from_hub, HUB_BASE_URL};
pub fn global_service() -> Router {
Router::new().route("/hub/list", get(list_hub_integrations))
@@ -24,14 +22,10 @@ async fn list_hub_integrations(
let (status_code, headers, response) = query_elems_from_hub(
&HTTP_CLIENT,
"https://hub.windmill.dev/integrations/list",
&format!("{}/integrations/list", *HUB_BASE_URL.read().await),
Some(query_params),
&db,
)
.await?;
Ok::<_, Error>((
status_code,
headers,
response
))
Ok::<_, Error>((status_code, headers, response))
}
backend/windmill-api/src/lib.rs
+5 -19
View File
@@ -171,18 +171,10 @@ pub async fn run_server(
let sp_extension = Arc::new(saml_ee::build_sp_extension().await?);
let embeddings_db = if server_mode {
if server_mode {
#[cfg(feature = "embedding")]
{
Some(load_embeddings_db(&db))
}
#[cfg(not(feature = "embedding"))]
{
Some(())
}
} else {
None
};
load_embeddings_db(&db)
}
let job_helpers_service = {
#[cfg(feature = "parquet")]
@@ -209,10 +201,7 @@ pub async fn run_server(
.nest("/apps", apps::workspaced_service())
.nest("/audit", audit::workspaced_service())
.nest("/capture", capture::workspaced_service())
.nest(
"/embeddings",
embeddings::workspaced_service(embeddings_db.clone()),
)
.nest("/embeddings", embeddings::workspaced_service())
.nest("/drafts", drafts::workspaced_service())
.nest("/favorites", favorite::workspaced_service())
.nest("/flows", flows::workspaced_service())
@@ -250,10 +239,7 @@ pub async fn run_server(
.nest("/flows", flows::global_service())
.nest("/apps", apps::global_service().layer(cors.clone()))
.nest("/schedules", schedule::global_service())
.nest(
"/embeddings",
embeddings::global_service(embeddings_db.clone()),
)
.nest("/embeddings", embeddings::global_service())
.route_layer(from_extractor::<ApiAuthed>())
.route_layer(from_extractor::<users::Tokened>())
.nest("/jobs", jobs::global_root_service())
backend/windmill-api/src/scripts.rs
+2 -1
View File
@@ -44,6 +44,7 @@ use windmill_common::{
utils::{
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
},
HUB_BASE_URL,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use windmill_parser_ts::remove_pinned_imports;
@@ -290,7 +291,7 @@ async fn get_top_hub_scripts(
let (status_code, headers, response) = query_elems_from_hub(
&HTTP_CLIENT,
"https://hub.windmill.dev/scripts/top",
&format!("{}/scripts/top", *HUB_BASE_URL.read().await),
Some(query_params),
&db,
)
backend/windmill-api/src/settings.rs
+2 -2
View File
@@ -26,7 +26,7 @@ use serde::Deserialize;
use tokio::time::timeout;
use windmill_common::{
error::{self, to_anyhow, JsonResult, Result},
global_settings::{AUTOMATE_USERNAME_CREATION_SETTING, ENV_SETTINGS},
global_settings::{AUTOMATE_USERNAME_CREATION_SETTING, ENV_SETTINGS, HUB_BASE_URL_SETTING},
server::Smtp,
};
@@ -147,7 +147,6 @@ pub async fn test_s3_bucket(
Ok("Tested blob storage successfully".to_string())
}
#[derive(Deserialize)]
pub struct TestKey {
pub license_key: String,
@@ -251,6 +250,7 @@ pub async fn get_global_setting(
if !key.starts_with("default_error_handler_")
&& !key.starts_with("default_recovery_handler_")
&& key != AUTOMATE_USERNAME_CREATION_SETTING
&& key != HUB_BASE_URL_SETTING
{
require_super_admin(&db, &authed.email).await?;
}
backend/windmill-common/src/global_settings.rs
+1
View File
@@ -23,6 +23,7 @@ pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexistin
pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config";
pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation";
pub const HUB_BASE_URL_SETTING: &str = "hub_base_url";
pub const ENV_SETTINGS: [&str; 50] = [
"DISABLE_NSJAIL",
backend/windmill-common/src/lib.rs
+4
View File
@@ -46,6 +46,8 @@ pub mod tracing_init;
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 4;
pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
lazy_static::lazy_static! {
pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT")
.ok()
@@ -69,6 +71,8 @@ lazy_static::lazy_static! {
pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
pub static ref HUB_BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new(DEFAULT_HUB_BASE_URL.to_string()));
}
pub async fn shutdown_signal(
backend/windmill-common/src/scripts.rs
+3 -3
View File
@@ -14,7 +14,7 @@ use std::{
use crate::{
error::{to_anyhow, Error},
utils::http_get_from_hub,
DB,
DB, HUB_BASE_URL,
};
use serde::de::Error as _;
use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize};
@@ -353,7 +353,7 @@ pub async fn get_hub_script_by_path(
let content = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/raw/{path}.ts"),
&format!("{}/raw/{}.ts", *HUB_BASE_URL.read().await, path),
true,
None,
db,
@@ -377,7 +377,7 @@ pub async fn get_full_hub_script_by_path(
let value = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/raw2/{path}"),
&format!("{}/raw2/{}", *HUB_BASE_URL.read().await, path),
true,
None,
db,
cli/hub.ts
+11 -9
View File
@@ -21,6 +21,11 @@ async function pull(opts: GlobalOptions) {
key: "uid",
});
const hubBaseUrl =
(await SettingService.getGlobal({
key: "hubBaseUrl",
})) ?? "https://hub.windmill.dev";
const headers = {
Accept: "application/json",
"X-email": userInfo.email,
@@ -40,20 +45,17 @@ async function pull(opts: GlobalOptions) {
created_by: string;
created_at: Date;
comments: never[];
}[] = await fetch("https://hub.windmill.dev/resource_types/list", {
}[] = await fetch(hubBaseUrl + "/resource_types/list", {
headers,
})
.then((r) => r.json())
.then((list: { id: number; name: string }[]) =>
list.map((x) =>
fetch(
"https://hub.windmill.dev/resource_types/" + x.id + "/" + x.name,
{
headers: {
Accept: "application/json",
},
}
)
fetch(hubBaseUrl + "/resource_types/" + x.id + "/" + x.name, {
headers: {
Accept: "application/json",
},
})
)
)
.then((x) => Promise.all(x))
frontend/src/lib/components/ErrorOrRecoveryHandler.svelte
+2 -1
View File
@@ -15,6 +15,7 @@
type Flow
} from '$lib/gen'
import { inferArgs } from '$lib/infer'
import { hubBaseUrlStore } from '$lib/stores'
import { CheckCircle2, Loader2, RotateCw, XCircle } from 'lucide-svelte'
@@ -221,7 +222,7 @@
<div class="text-xs">
Example of error handler scripts can be found on <a
target="_blank"
href="https://hub.windmill.dev/failures"
href="{$hubBaseUrlStore}/failures"
>
Windmill Hub</a
>
frontend/src/lib/components/FlowGraphViewerStep.svelte
+5 -4
View File
@@ -11,6 +11,7 @@
import SchemaViewer from './SchemaViewer.svelte'
import { scriptPathToHref } from '$lib/scripts'
import { cleanExpr } from '$lib/utils'
import { hubBaseUrlStore } from '$lib/stores'
import { twMerge } from 'tailwind-merge'
import FlowModuleScript from './flows/content/FlowModuleScript.svelte'
@@ -34,7 +35,7 @@
<a
rel="noreferrer"
target="_blank"
href={scriptPathToHref(stepDetail?.value?.path ?? '')}
href={scriptPathToHref(stepDetail?.value?.path ?? '', $hubBaseUrlStore)}
class=""
>
<IconedPath path={stepDetail?.value?.path ?? ''} />
@@ -52,7 +53,7 @@
class="w-full h-full text-sm"
title="embedded script from hub"
frameborder="0"
src="https://hub.windmill.dev/embed/script/{stepDetail.value?.path?.substring(4)}"
src="{$hubBaseUrlStore}/embed/script/{stepDetail.value?.path?.substring(4)}"
/>
</div>
{/if}
@@ -116,7 +117,7 @@
<a
rel="noreferrer"
target="_blank"
href={scriptPathToHref(stepDetail?.value?.path ?? '')}
href={scriptPathToHref(stepDetail?.value?.path ?? '', $hubBaseUrlStore)}
class=""
>
<IconedPath path={stepDetail?.value?.path ?? ''} />
@@ -166,7 +167,7 @@
class="w-full grow text-sm"
title="embedded script from hub"
frameborder="0"
src="https://hub.windmill.dev/embed/script/{stepDetail.value?.path?.substring(4)}"
src="{$hubBaseUrlStore}/embed/script/{stepDetail.value?.path?.substring(4)}"
/>
</div>
{:else}
frontend/src/lib/components/apps/editor/AppEditorHeader.svelte
+1 -1
View File
@@ -628,7 +628,7 @@
// displayName: 'Publish to Hub',
// icon: faGlobe,
// action: () => {
// const url = appToHubUrl(toStatic($app, $staticExporter, $summary))
// const url = appToHubUrl(toStatic($app, $staticExporter, $summary, $hubBaseUrlStore))
// window.open(url.toString(), '_blank')
// }
// },
frontend/src/lib/components/instanceSettings.ts
+8
View File
@@ -110,6 +110,14 @@ export const settings: Record<string, Setting[]> = {
fieldType: 'text',
storage: 'setting',
ee_only: ''
},
{
label: 'Private hub base url',
description: 'Base url of your private hub instance',
key: 'hub_base_url',
fieldType: 'text',
storage: 'setting',
ee_only: ''
}
],
'SSO/OAuth': [],
frontend/src/lib/hub.ts
+7 -6
View File
@@ -9,9 +9,10 @@ export function scriptToHubUrl(
kind: Script.kind,
language: Script.language,
schema: Schema | any,
lock: string | undefined
lock: string | undefined,
hubBaseUrl: string
): URL {
const url = new URL('https://hub.windmill.dev/scripts/add')
const url = new URL(hubBaseUrl + '/scripts/add')
url.hash = encodeState({ content, summary, description, kind, language, schema, lock })
return url
@@ -37,8 +38,8 @@ export async function loadHubApps() {
}
}
export function flowToHubUrl(flow: Flow): URL {
const url = new URL('https://hub.windmill.dev/flows/add')
export function flowToHubUrl(flow: Flow, hubBaseUrl: string): URL {
const url = new URL(hubBaseUrl + '/flows/add')
const openFlow = {
value: flow.value,
summary: flow.summary,
@@ -49,8 +50,8 @@ export function flowToHubUrl(flow: Flow): URL {
return url
}
export function appToHubUrl(staticApp: any): URL {
const url = new URL('https://hub.windmill.dev/apps/add')
export function appToHubUrl(staticApp: any, hubBaseUrl: string): URL {
const url = new URL(hubBaseUrl + '/apps/add')
url.searchParams.append('app', encodeState(staticApp))
return url
}
frontend/src/lib/scripts.ts
+2 -2
View File
@@ -79,9 +79,9 @@ export async function loadSchemaFlow(path: string): Promise<Schema> {
return flow.schema as any
}
export function scriptPathToHref(path: string): string {
export function scriptPathToHref(path: string, hubBaseUrl: string): string {
if (path.startsWith('hub/')) {
return 'https://hub.windmill.dev/from_version/' + path.substring(4)
return hubBaseUrl + '/from_version/' + path.substring(4)
} else {
return `/scripts/get/${path}?workspace=${get(workspaceStore)}`
}
frontend/src/lib/stores.ts
+1
View File
@@ -47,6 +47,7 @@ export const starStore = writable(1)
export const usersWorkspaceStore = writable<UserWorkspaceList | undefined>(undefined)
export const superadmin = writable<string | false | undefined>(undefined)
export const lspTokenStore = writable<string | undefined>(undefined)
export const hubBaseUrlStore = writable<string>('https://hub.windmill.dev')
export const userWorkspaces: Readable<
Array<{
id: string
frontend/src/routes/(root)/(logged)/+layout.svelte
+9 -1
View File
@@ -7,6 +7,7 @@
OpenAPI,
RawAppService,
ScriptService,
SettingService,
UserService,
WorkspaceService
} from '$lib/gen'
@@ -23,7 +24,8 @@
userStore,
workspaceStore,
type UserExt,
defaultScripts
defaultScripts,
hubBaseUrlStore
} from '$lib/stores'
import CenteredModal from '$lib/components/CenteredModal.svelte'
import { afterNavigate, beforeNavigate, goto } from '$app/navigation'
@@ -113,6 +115,7 @@
loadFavorites()
loadUsage()
syncTutorialsTodos()
loadHubBaseUrl()
}
async function loadUsage() {
@@ -124,6 +127,11 @@
}
}
async function loadHubBaseUrl() {
$hubBaseUrlStore =
(await SettingService.getGlobal({ key: 'hub_base_url' })) ?? 'https://hub.windmill.dev'
}
async function loadFavorites() {
const scripts = await ScriptService.listScripts({
workspace: $workspaceStore ?? '',
frontend/src/routes/(root)/(logged)/+page.svelte
+7 -7
View File
@@ -12,6 +12,7 @@
import FlowViewer from '$lib/components/FlowViewer.svelte'
import HighlightCode from '$lib/components/HighlightCode.svelte'
import { Building, ExternalLink, GitFork, Globe2, Loader2 } from 'lucide-svelte'
import { hubBaseUrlStore } from '$lib/stores'
import ItemsList from '$lib/components/home/ItemsList.svelte'
import CreateActionsApp from '$lib/components/flows/CreateActionsApp.svelte'
@@ -82,8 +83,7 @@
<DrawerContent title={codeViewerObj?.summary ?? ''} on:close={codeViewer.closeDrawer}>
<svelte:fragment slot="actions">
<Button
href="https://hub.windmill.dev/scripts/{codeViewerObj?.app ?? ''}/{codeViewerObj?.ask_id ??
0}"
href="{$hubBaseUrlStore}/scripts/{codeViewerObj?.app ?? ''}/{codeViewerObj?.ask_id ?? 0}"
variant="contained"
color="light"
size="xs"
@@ -119,7 +119,7 @@
<DrawerContent title="Hub flow" on:close={flowViewer.closeDrawer}>
<svelte:fragment slot="actions">
<Button
href="https://hub.windmill.dev/flows/{flowViewerFlow?.flow?.id}"
href="{$hubBaseUrlStore}/flows/{flowViewerFlow?.flow?.id}"
variant="contained"
color="light"
size="xs"
@@ -157,7 +157,7 @@
<DrawerContent title="Hub app" on:close={appViewer.closeDrawer}>
<svelte:fragment slot="actions">
<Button
href="https://hub.windmill.dev/apps/{appViewerApp?.app?.id}"
href="{$hubBaseUrlStore}/apps/{appViewerApp?.app?.id}"
variant="contained"
color="light"
size="xs"
@@ -271,7 +271,7 @@
<Button
startIcon={{ icon: ExternalLink }}
target="_blank"
href="https://hub.windmill.dev"
href={$hubBaseUrlStore}
variant="border"
color="light"
>
@@ -283,7 +283,7 @@
<Button
startIcon={{ icon: ExternalLink }}
target="_blank"
href="https://hub.windmill.dev"
href={$hubBaseUrlStore}
variant="border"
color="light"
>Hub
@@ -294,7 +294,7 @@
<Button
startIcon={{ icon: ExternalLink }}
target="_blank"
href="https://hub.windmill.dev"
href={$hubBaseUrlStore}
variant="border"
color="light">Hub</Button
>
frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte
+3 -2
View File
@@ -10,7 +10,7 @@
} from '$lib/utils'
import Tooltip from '$lib/components/Tooltip.svelte'
import ShareModal from '$lib/components/ShareModal.svelte'
import { runFormStore, userStore, workspaceStore } from '$lib/stores'
import { hubBaseUrlStore, runFormStore, userStore, workspaceStore } from '$lib/stores'
import SchemaViewer from '$lib/components/SchemaViewer.svelte'
import { onDestroy } from 'svelte'
import HighlightCode from '$lib/components/HighlightCode.svelte'
@@ -374,7 +374,8 @@
script.kind,
script.language,
script.schema,
script.lock ?? ''
script.lock ?? '',
$hubBaseUrlStore
).toString(),
'_blank'
)
frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+4 -5
View File
@@ -29,7 +29,8 @@
superadmin,
userStore,
usersWorkspaceStore,
workspaceStore
workspaceStore,
hubBaseUrlStore
} from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { setQueryWithoutLoad, emptyString, tryEvery } from '$lib/utils'
@@ -799,10 +800,8 @@
<div class="prose text-2xs text-tertiary">
Pick a script or flow meant to be triggered when the `/windmill` command is invoked. Upon
connection, templates for a <a href="https://hub.windmill.dev/scripts/slack/1405/"
>script</a
>
and <a href="https://hub.windmill.dev/flows/28/">flow</a> are available.
connection, templates for a <a href="{$hubBaseUrlStore}/scripts/slack/1405/">script</a>
and <a href="{$hubBaseUrlStore}/flows/28/">flow</a> are available.
<br /><br />