diff --git a/Dockerfile b/Dockerfile index 7fd89ba6fa..c0f15162de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -175,7 +175,11 @@ WORKDIR ${APP} RUN ln -s ${APP}/windmill /usr/local/bin/windmill -RUN windmill cache +COPY ./frontend/src/lib/hubPaths.json ${APP}/hubPaths.json + +RUN windmill cache ${APP}/hubPaths.json + +RUN rm ${APP}/hubPaths.json EXPOSE 8000 diff --git a/backend/src/main.rs b/backend/src/main.rs index c26695d7c5..ebc7f6dc3b 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -6,15 +6,20 @@ * LICENSE-AGPL for a copy of the license. */ +use anyhow::Context; use gethostname::gethostname; use git_version::git_version; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; use std::{ + collections::HashMap, net::{IpAddr, Ipv4Addr, SocketAddr}, time::Duration, }; -use tokio::fs::DirBuilder; +use tokio::{ + fs::{create_dir_all, DirBuilder, File}, + io::AsyncReadExt, +}; use windmill_api::HTTP_CLIENT; #[cfg(feature = "enterprise")] @@ -54,9 +59,10 @@ use windmill_common::METRICS_ADDR; use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ - BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, - DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR, - LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, + get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, + BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, + GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, + POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, }; use crate::monitor::{ @@ -111,6 +117,28 @@ pub fn main() -> anyhow::Result<()> { create_and_run_current_thread_inner(windmill_main()) } +async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { + let file_path = file_path.unwrap_or("./hubPaths.json".to_string()); + let mut file = File::open(&file_path) + .await + .with_context(|| format!("Could not open {}, make sure it exists", &file_path))?; + let mut contents = String::new(); + file.read_to_string(&mut contents).await?; + let paths = serde_json::from_str::>(&contents).with_context(|| { + format!( + "Could not parse {}, make sure it is a valid JSON object with string keys and values", + &file_path + ) + })?; + + create_dir_all(HUB_CACHE_DIR).await?; + + for path in paths.values() { + get_hub_script_content_and_requirements(Some(path.to_string()), None).await?; + } + Ok(()) +} + async fn windmill_main() -> anyhow::Result<()> { dotenv::dotenv().ok(); @@ -141,6 +169,9 @@ async fn windmill_main() -> anyhow::Result<()> { { tracing::warn!("Embeddings are not enabled, ignoring..."); } + + cache_hub_scripts(std::env::args().nth(2)).await?; + return Ok(()); } "-v" | "--version" | "version" => { diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 97409f5b81..9bfb84e34f 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -705,7 +705,7 @@ pub async fn get_hub_app_by_id( &format!("{}/apps/{}/json", *HUB_BASE_URL.read().await, id), false, None, - &db, + Some(&db), ) .await? .json() diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 6848ceb3d3..5cc7226753 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -313,7 +313,7 @@ impl EmbeddingsDb { &format!("{}/scripts/embeddings", hub_base_url), false, None, - pg_db, + Some(pg_db), ) .await? } else { @@ -326,7 +326,7 @@ impl EmbeddingsDb { &format!("{}/scripts/embeddings", hub_base_url), false, None, - pg_db, + Some(pg_db), ) .await? } @@ -375,7 +375,7 @@ impl EmbeddingsDb { &format!("{}/resource_types/embeddings", hub_base_url), false, None, - pg_db, + Some(pg_db), ) .await? } else { @@ -388,7 +388,7 @@ impl EmbeddingsDb { &format!("{}/resource_types/embeddings", hub_base_url), false, None, - pg_db, + Some(pg_db), ) .await? } diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index c1f8ec70a4..ccec53c4dc 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -230,7 +230,7 @@ pub async fn get_hub_flow_by_id( &format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id), false, None, - &db, + Some(&db), ) .await? .json() diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 0ed84404a8..41214dfcbb 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -817,7 +817,8 @@ pub async fn get_full_hub_script_by_path( Extension(db): Extension, ) -> JsonResult { Ok(Json( - windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, &db).await?, + windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, Some(&db)) + .await?, )) } diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 86c506f467..b10558e40f 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -384,7 +384,7 @@ pub async fn get_hub_script_by_path( &format!("{}/raw/{}.ts", hub_base_url, path), true, None, - db, + Some(db), ) .await? .text() @@ -409,7 +409,7 @@ pub async fn get_hub_script_by_path( &format!("{}/raw/{}.ts", DEFAULT_HUB_BASE_URL, path), true, None, - db, + Some(db), ) .await? .text() @@ -427,7 +427,7 @@ pub async fn get_hub_script_by_path( pub async fn get_full_hub_script_by_path( path: StripPath, http_client: &reqwest::Client, - db: &DB, + db: Option<&DB>, ) -> crate::error::Result { let path = path .to_path() diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 81031a7359..63cece0fe2 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -96,7 +96,7 @@ pub async fn query_elems_from_hub( reqwest::header::HeaderMap, axum::body::Body, )> { - let response = http_get_from_hub(http_client, url, false, query_params, db).await?; + let response = http_get_from_hub(http_client, url, false, query_params, Some(db)).await?; let status = response.status(); @@ -112,9 +112,18 @@ pub async fn http_get_from_hub( url: &str, plain: bool, query_params: Option>, - db: &Pool, + db: Option<&Pool>, ) -> Result { - let uid = get_uid(db).await; + let uid = match db { + Some(db) => match get_uid(db).await { + Ok(uid) => Some(uid), + Err(err) => { + tracing::info!("No valid uid found: {}", err); + None + } + }, + None => None, + }; let mut request = http_client.get(url).header( "Accept", @@ -125,10 +134,8 @@ pub async fn http_get_from_hub( }, ); - if let Ok(uid) = uid { + if let Some(uid) = uid { request = request.header("X-uid", uid); - } else { - tracing::info!("No valid uid found: {}", uid.err().unwrap()) } if let Some(query_params) = query_params { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 4214398752..7e1bf9b192 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -53,7 +53,6 @@ use windmill_common::{ jobs::{ get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode, }, - oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, schedule::Schedule, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, @@ -1479,16 +1478,6 @@ pub async fn push_error_handler< } } - // TODO(gbouv): REMOVE THIS after December 1st 2023 and ping users to re-save their error handlers - if on_failure_path - .to_string() - .eq("script/hub/5792/workspace-or-schedule-error-handler-slack") - { - // default slack error handler being used -> we need to inject the slack token - let slack_resource = format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}"); - extra.insert("slack".to_string(), to_raw_value(&slack_resource)); - } - let result = sanitize_result(result); let tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq); @@ -1602,15 +1591,6 @@ async fn handle_recovered_schedule< )); } } - // TODO(gbouv): REMOVE THIS after December 1st 2023 and ping users to re-save their error handlers - if on_recovery_path - .to_string() - .eq("script/hub/2430/slack/schedule-recovery-handler-slack") - { - // default slack error handler being used -> we need to inject the slack token - let slack_resource = format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}"); - extra.insert("slack".to_string(), to_raw_value(&slack_resource)); - } let args = error_job .result @@ -3137,7 +3117,8 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( } let hub_script = - get_full_hub_script_by_path(StripPath(path.clone()), &HTTP_CLIENT, _db).await?; + get_full_hub_script_by_path(StripPath(path.clone()), &HTTP_CLIENT, Some(_db)) + .await?; ( None, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b34fede376..6a6ccc7daf 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2736,9 +2736,9 @@ pub struct ContentReqLangEnvs { pub codebase: Option, } -async fn get_hub_script_content_and_requirements( +pub async fn get_hub_script_content_and_requirements( script_path: Option, - db: &DB, + db: Option<&DB>, ) -> error::Result { let script_path = script_path .clone() @@ -2783,7 +2783,7 @@ async fn get_script_content_by_path( .clone() .ok_or_else(|| Error::InternalErr(format!("expected script path")))?; return if script_path.starts_with("hub/") { - get_hub_script_content_and_requirements(Some(script_path), db).await + get_hub_script_content_and_requirements(Some(script_path), Some(db)).await } else { let (script_hash, ..) = get_latest_deployed_hash_for_path(db, w_id, script_path.as_str()).await?; @@ -2870,7 +2870,7 @@ async fn handle_code_execution_job( codebase }}, JobKind::Script_Hub => { - get_hub_script_content_and_requirements(job.script_path.clone(), db).await? + get_hub_script_content_and_requirements(job.script_path.clone(), Some(db)).await? } JobKind::Script => { get_script_content_by_hash( diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index af27740cd6..47d26b0470 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -27,7 +27,11 @@ COPY --from=denoland/deno:1.44.4 --chmod=755 /usr/bin/deno /usr/bin/deno RUN ln -s ${APP}/windmill /usr/local/bin/windmill -RUN windmill cache +COPY ./frontend/src/lib/hubPaths.json ${APP}/hubPaths.json + +RUN windmill cache ${APP}/hubPaths.json + +RUN rm ${APP}/hubPaths.json EXPOSE 8000 diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index 465805fb61..c54965d8be 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -19,9 +19,10 @@ import { hubBaseUrlStore } from '$lib/stores' import { CheckCircle2, Loader2, RotateCw, XCircle } from 'lucide-svelte' + import { hubPaths } from '$lib/hub' - const slackRecoveryHandler = 'hub/2430/slack/schedule-recovery-handler-slack' - const slackHandlerScriptPath = 'hub/6512/workspace-or-schedule-error-handler-slack' + const slackRecoveryHandler = hubPaths.slackRecoveryHandler + const slackHandlerScriptPath = hubPaths.slackErrorHandler export let errorOrRecovery: 'error' | 'recovery' export let isEditable: boolean diff --git a/frontend/src/lib/components/apps/editor/AppReportsDrawer.svelte b/frontend/src/lib/components/apps/editor/AppReportsDrawer.svelte index c2c160722e..5df6e41fb2 100644 --- a/frontend/src/lib/components/apps/editor/AppReportsDrawer.svelte +++ b/frontend/src/lib/components/apps/editor/AppReportsDrawer.svelte @@ -24,6 +24,7 @@ import { RotateCw, Save } from 'lucide-svelte' import { CUSTOM_TAGS_SETTING, WORKSPACE_SLACK_BOT_TOKEN_PATH } from '$lib/consts' import { loadSchemaFromPath } from '$lib/infer' + import { hubPaths } from '$lib/hub' export let appPath: string export let open = false @@ -187,7 +188,7 @@ export async function main(app_path: string, startup_duration = 5, kind: 'pdf' | const notificationScripts = { discord: { - path: 'hub/7838/discord', + path: hubPaths.discordReport, schema: { type: 'object', properties: { @@ -203,7 +204,7 @@ export async function main(app_path: string, startup_duration = 5, kind: 'pdf' | } }, slack: { - path: 'hub/7836/slack', // if to be updated, also update it in in backend/windmill-queue/src/jobs.rs + path: hubPaths.slackReport, // if to be updated, also update it in in backend/windmill-queue/src/jobs.rs schema: { type: 'object', properties: { @@ -216,7 +217,7 @@ export async function main(app_path: string, startup_duration = 5, kind: 'pdf' | } }, email: { - path: 'hub/7837/smtp', + path: hubPaths.smtpReport, schema: { type: 'object', properties: { diff --git a/frontend/src/lib/hub.ts b/frontend/src/lib/hub.ts index cb4fba3f2a..f5ff47ff1a 100644 --- a/frontend/src/lib/hub.ts +++ b/frontend/src/lib/hub.ts @@ -1,6 +1,7 @@ import type { Schema } from './common' import { AppService, FlowService, type Flow, type Script } from './gen' import { encodeState } from './utils' +import rawHubPaths from './hubPaths.json?raw' export function scriptToHubUrl( content: string, @@ -55,3 +56,15 @@ export function appToHubUrl(staticApp: any, hubBaseUrl: string): URL { url.searchParams.append('app', encodeState(staticApp)) return url } + +type HubPaths = { + gitSync: string + gitSyncTest: string + slackErrorHandler: string + slackRecoveryHandler: string + slackReport: string + discordReport: string + smtpReport: string +} + +export const hubPaths = JSON.parse(rawHubPaths) as HubPaths diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json new file mode 100644 index 0000000000..5d274614a0 --- /dev/null +++ b/frontend/src/lib/hubPaths.json @@ -0,0 +1,9 @@ +{ + "gitSync": "hub/8931/sync-script-to-git-repo-windmill", + "gitSyncTest": "hub/8944/git-repo-test-read-write-windmill", + "slackErrorHandler": "hub/6512/workspace-or-schedule-error-handler-slack", + "slackRecoveryHandler": "hub/2430/slack/schedule-recovery-handler-slack", + "slackReport": "hub/7836/slack", + "discordReport": "hub/7838/discord", + "smtpReport": "hub/7837/smtp" +} \ No newline at end of file diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 723336e2a3..a06bed1d34 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -55,6 +55,7 @@ type S3ResourceSettings } from '$lib/workspace_settings' import { base } from '$lib/base' + import { hubPaths } from '$lib/hub' type GitSyncTypeMap = { scripts: boolean @@ -138,7 +139,7 @@ | 'error_handler') ?? 'users' let usingOpenaiClientCredentialsOauth = false - const latestGitSyncHubScript = `hub/8931/sync-script-to-git-repo-windmill` + const latestGitSyncHubScript = hubPaths.gitSync // function getDropDownItems(username: string): DropdownItem[] { // return [ // { @@ -566,7 +567,7 @@ } let jobId = await JobService.runScriptByPath({ workspace: $workspaceStore!, - path: 'hub/8944/git-repo-test-read-write-windmill', + path: hubPaths.gitSyncTest, requestBody: { repo_url_resource_path: gitSyncRepository.git_repo_resource_path.replace('$res:', '') }