starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-23 08:00:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: cache common hub scripts in image (#4249)

Browse Source 
* feat: cache common hub scripts in image

* Delete backend/src/hubPaths.json

* fix: missing file

* fix: dockerfile

* fix: cache by path

* fix

* fix

* precreate cache folder
This commit is contained in:
HugoCasa
2024-08-16 17:30:44 +02:00
committed by GitHub
parent 5a8fa1d724
commit 99f7828ebb
16 changed files with 107 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+5 -1
View File
@@ -175,7 +175,11 @@ WORKDIR ${APP}
RUN ln -s ${APP}/windmill /usr/local/bin/windmill
RUN windmill cache
COPY ./frontend/src/lib/hubPaths.json ${APP}/hubPaths.json
RUN windmill cache ${APP}/hubPaths.json
RUN rm ${APP}/hubPaths.json
EXPOSE 8000
backend/src/main.rs
+35 -4
View File
@@ -6,15 +6,20 @@
* LICENSE-AGPL for a copy of the license.
*/
use anyhow::Context;
use gethostname::gethostname;
use git_version::git_version;
use rand::Rng;
use sqlx::{postgres::PgListener, Pool, Postgres};
use std::{
collections::HashMap,
net::{IpAddr, Ipv4Addr, SocketAddr},
time::Duration,
};
use tokio::fs::DirBuilder;
use tokio::{
fs::{create_dir_all, DirBuilder, File},
io::AsyncReadExt,
};
use windmill_api::HTTP_CLIENT;
#[cfg(feature = "enterprise")]
@@ -54,9 +59,10 @@ use windmill_common::METRICS_ADDR;
use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_worker::{
BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR,
DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR,
LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR,
POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
};
use crate::monitor::{
@@ -111,6 +117,28 @@ pub fn main() -> anyhow::Result<()> {
create_and_run_current_thread_inner(windmill_main())
}
async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
let file_path = file_path.unwrap_or("./hubPaths.json".to_string());
let mut file = File::open(&file_path)
.await
.with_context(|| format!("Could not open {}, make sure it exists", &file_path))?;
let mut contents = String::new();
file.read_to_string(&mut contents).await?;
let paths = serde_json::from_str::<HashMap<String, String>>(&contents).with_context(|| {
format!(
"Could not parse {}, make sure it is a valid JSON object with string keys and values",
&file_path
)
})?;
create_dir_all(HUB_CACHE_DIR).await?;
for path in paths.values() {
get_hub_script_content_and_requirements(Some(path.to_string()), None).await?;
}
Ok(())
}
async fn windmill_main() -> anyhow::Result<()> {
dotenv::dotenv().ok();
@@ -141,6 +169,9 @@ async fn windmill_main() -> anyhow::Result<()> {
{
tracing::warn!("Embeddings are not enabled, ignoring...");
}
cache_hub_scripts(std::env::args().nth(2)).await?;
return Ok(());
}
"-v" | "--version" | "version" => {
backend/windmill-api/src/apps.rs
+1 -1
View File
@@ -705,7 +705,7 @@ pub async fn get_hub_app_by_id(
&format!("{}/apps/{}/json", *HUB_BASE_URL.read().await, id),
false,
None,
&db,
Some(&db),
)
.await?
.json()
backend/windmill-api/src/embeddings.rs
+4 -4
View File
@@ -313,7 +313,7 @@ impl EmbeddingsDb {
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
pg_db,
Some(pg_db),
)
.await?
} else {
@@ -326,7 +326,7 @@ impl EmbeddingsDb {
&format!("{}/scripts/embeddings", hub_base_url),
false,
None,
pg_db,
Some(pg_db),
)
.await?
}
@@ -375,7 +375,7 @@ impl EmbeddingsDb {
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
pg_db,
Some(pg_db),
)
.await?
} else {
@@ -388,7 +388,7 @@ impl EmbeddingsDb {
&format!("{}/resource_types/embeddings", hub_base_url),
false,
None,
pg_db,
Some(pg_db),
)
.await?
}
backend/windmill-api/src/flows.rs
+1 -1
View File
@@ -230,7 +230,7 @@ pub async fn get_hub_flow_by_id(
&format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id),
false,
None,
&db,
Some(&db),
)
.await?
.json()
backend/windmill-api/src/scripts.rs
+2 -1
View File
@@ -817,7 +817,8 @@ pub async fn get_full_hub_script_by_path(
Extension(db): Extension<DB>,
) -> JsonResult<HubScript> {
Ok(Json(
windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, &db).await?,
windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, Some(&db))
.await?,
))
}
backend/windmill-common/src/scripts.rs
+3 -3
View File
@@ -384,7 +384,7 @@ pub async fn get_hub_script_by_path(
&format!("{}/raw/{}.ts", hub_base_url, path),
true,
None,
db,
Some(db),
)
.await?
.text()
@@ -409,7 +409,7 @@ pub async fn get_hub_script_by_path(
&format!("{}/raw/{}.ts", DEFAULT_HUB_BASE_URL, path),
true,
None,
db,
Some(db),
)
.await?
.text()
@@ -427,7 +427,7 @@ pub async fn get_hub_script_by_path(
pub async fn get_full_hub_script_by_path(
path: StripPath,
http_client: &reqwest::Client,
db: &DB,
db: Option<&DB>,
) -> crate::error::Result<HubScript> {
let path = path
.to_path()
backend/windmill-common/src/utils.rs
+13 -6
View File
@@ -96,7 +96,7 @@ pub async fn query_elems_from_hub(
reqwest::header::HeaderMap,
axum::body::Body,
)> {
let response = http_get_from_hub(http_client, url, false, query_params, db).await?;
let response = http_get_from_hub(http_client, url, false, query_params, Some(db)).await?;
let status = response.status();
@@ -112,9 +112,18 @@ pub async fn http_get_from_hub(
url: &str,
plain: bool,
query_params: Option<Vec<(&str, String)>>,
db: &Pool<Postgres>,
db: Option<&Pool<Postgres>>,
) -> Result<reqwest::Response> {
let uid = get_uid(db).await;
let uid = match db {
Some(db) => match get_uid(db).await {
Ok(uid) => Some(uid),
Err(err) => {
tracing::info!("No valid uid found: {}", err);
None
}
},
None => None,
};
let mut request = http_client.get(url).header(
"Accept",
@@ -125,10 +134,8 @@ pub async fn http_get_from_hub(
},
);
if let Ok(uid) = uid {
if let Some(uid) = uid {
request = request.header("X-uid", uid);
} else {
tracing::info!("No valid uid found: {}", uid.err().unwrap())
}
if let Some(query_params) = query_params {
backend/windmill-queue/src/jobs.rs
+2 -21
View File
@@ -53,7 +53,6 @@ use windmill_common::{
jobs::{
get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode,
},
oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH,
schedule::Schedule,
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
@@ -1479,16 +1478,6 @@ pub async fn push_error_handler<
}
}
// TODO(gbouv): REMOVE THIS after December 1st 2023 and ping users to re-save their error handlers
if on_failure_path
.to_string()
.eq("script/hub/5792/workspace-or-schedule-error-handler-slack")
{
// default slack error handler being used -> we need to inject the slack token
let slack_resource = format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}");
extra.insert("slack".to_string(), to_raw_value(&slack_resource));
}
let result = sanitize_result(result);
let tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
@@ -1602,15 +1591,6 @@ async fn handle_recovered_schedule<
));
}
}
// TODO(gbouv): REMOVE THIS after December 1st 2023 and ping users to re-save their error handlers
if on_recovery_path
.to_string()
.eq("script/hub/2430/slack/schedule-recovery-handler-slack")
{
// default slack error handler being used -> we need to inject the slack token
let slack_resource = format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}");
extra.insert("slack".to_string(), to_raw_value(&slack_resource));
}
let args = error_job
.result
@@ -3137,7 +3117,8 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
}
let hub_script =
get_full_hub_script_by_path(StripPath(path.clone()), &HTTP_CLIENT, _db).await?;
get_full_hub_script_by_path(StripPath(path.clone()), &HTTP_CLIENT, Some(_db))
.await?;
(
None,
backend/windmill-worker/src/worker.rs
+4 -4
View File
@@ -2736,9 +2736,9 @@ pub struct ContentReqLangEnvs {
pub codebase: Option<String>,
}
async fn get_hub_script_content_and_requirements(
pub async fn get_hub_script_content_and_requirements(
script_path: Option<String>,
db: &DB,
db: Option<&DB>,
) -> error::Result<ContentReqLangEnvs> {
let script_path = script_path
.clone()
@@ -2783,7 +2783,7 @@ async fn get_script_content_by_path(
.clone()
.ok_or_else(|| Error::InternalErr(format!("expected script path")))?;
return if script_path.starts_with("hub/") {
get_hub_script_content_and_requirements(Some(script_path), db).await
get_hub_script_content_and_requirements(Some(script_path), Some(db)).await
} else {
let (script_hash, ..) =
get_latest_deployed_hash_for_path(db, w_id, script_path.as_str()).await?;
@@ -2870,7 +2870,7 @@ async fn handle_code_execution_job(
codebase
}},
JobKind::Script_Hub => {
get_hub_script_content_and_requirements(job.script_path.clone(), db).await?
get_hub_script_content_and_requirements(job.script_path.clone(), Some(db)).await?
}
JobKind::Script => {
get_script_content_by_hash(
docker/DockerfileSlim
+5 -1
View File
@@ -27,7 +27,11 @@ COPY --from=denoland/deno:1.44.4 --chmod=755 /usr/bin/deno /usr/bin/deno
RUN ln -s ${APP}/windmill /usr/local/bin/windmill
RUN windmill cache
COPY ./frontend/src/lib/hubPaths.json ${APP}/hubPaths.json
RUN windmill cache ${APP}/hubPaths.json
RUN rm ${APP}/hubPaths.json
EXPOSE 8000
frontend/src/lib/components/ErrorOrRecoveryHandler.svelte
+3 -2
View File
@@ -19,9 +19,10 @@
import { hubBaseUrlStore } from '$lib/stores'
import { CheckCircle2, Loader2, RotateCw, XCircle } from 'lucide-svelte'
import { hubPaths } from '$lib/hub'
const slackRecoveryHandler = 'hub/2430/slack/schedule-recovery-handler-slack'
const slackHandlerScriptPath = 'hub/6512/workspace-or-schedule-error-handler-slack'
const slackRecoveryHandler = hubPaths.slackRecoveryHandler
const slackHandlerScriptPath = hubPaths.slackErrorHandler
export let errorOrRecovery: 'error' | 'recovery'
export let isEditable: boolean
frontend/src/lib/components/apps/editor/AppReportsDrawer.svelte
+4 -3
View File
@@ -24,6 +24,7 @@
import { RotateCw, Save } from 'lucide-svelte'
import { CUSTOM_TAGS_SETTING, WORKSPACE_SLACK_BOT_TOKEN_PATH } from '$lib/consts'
import { loadSchemaFromPath } from '$lib/infer'
import { hubPaths } from '$lib/hub'
export let appPath: string
export let open = false
@@ -187,7 +188,7 @@ export async function main(app_path: string, startup_duration = 5, kind: 'pdf' |
const notificationScripts = {
discord: {
path: 'hub/7838/discord',
path: hubPaths.discordReport,
schema: {
type: 'object',
properties: {
@@ -203,7 +204,7 @@ export async function main(app_path: string, startup_duration = 5, kind: 'pdf' |
}
},
slack: {
path: 'hub/7836/slack', // if to be updated, also update it in in backend/windmill-queue/src/jobs.rs
path: hubPaths.slackReport, // if to be updated, also update it in in backend/windmill-queue/src/jobs.rs
schema: {
type: 'object',
properties: {
@@ -216,7 +217,7 @@ export async function main(app_path: string, startup_duration = 5, kind: 'pdf' |
}
},
email: {
path: 'hub/7837/smtp',
path: hubPaths.smtpReport,
schema: {
type: 'object',
properties: {
frontend/src/lib/hub.ts
+13
View File
@@ -1,6 +1,7 @@
import type { Schema } from './common'
import { AppService, FlowService, type Flow, type Script } from './gen'
import { encodeState } from './utils'
import rawHubPaths from './hubPaths.json?raw'
export function scriptToHubUrl(
content: string,
@@ -55,3 +56,15 @@ export function appToHubUrl(staticApp: any, hubBaseUrl: string): URL {
url.searchParams.append('app', encodeState(staticApp))
return url
}
type HubPaths = {
gitSync: string
gitSyncTest: string
slackErrorHandler: string
slackRecoveryHandler: string
slackReport: string
discordReport: string
smtpReport: string
}
export const hubPaths = JSON.parse(rawHubPaths) as HubPaths
frontend/src/lib/hubPaths.json
+9
View File
@@ -0,0 +1,9 @@
{
"gitSync": "hub/8931/sync-script-to-git-repo-windmill",
"gitSyncTest": "hub/8944/git-repo-test-read-write-windmill",
"slackErrorHandler": "hub/6512/workspace-or-schedule-error-handler-slack",
"slackRecoveryHandler": "hub/2430/slack/schedule-recovery-handler-slack",
"slackReport": "hub/7836/slack",
"discordReport": "hub/7838/discord",
"smtpReport": "hub/7837/smtp"
}
frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+3 -2
View File
@@ -55,6 +55,7 @@
type S3ResourceSettings
} from '$lib/workspace_settings'
import { base } from '$lib/base'
import { hubPaths } from '$lib/hub'
type GitSyncTypeMap = {
scripts: boolean
@@ -138,7 +139,7 @@
| 'error_handler') ?? 'users'
let usingOpenaiClientCredentialsOauth = false
const latestGitSyncHubScript = `hub/8931/sync-script-to-git-repo-windmill`
const latestGitSyncHubScript = hubPaths.gitSync
// function getDropDownItems(username: string): DropdownItem[] {
// return [
// {
@@ -566,7 +567,7 @@
}
let jobId = await JobService.runScriptByPath({
workspace: $workspaceStore!,
path: 'hub/8944/git-repo-test-read-write-windmill',
path: hubPaths.gitSyncTest,
requestBody: {
repo_url_resource_path: gitSyncRepository.git_repo_resource_path.replace('$res:', '')
}