From b076e093df351ae3a0d3507a69121ebb0335513a Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Mon, 23 Oct 2023 17:05:54 +0200 Subject: [PATCH] feat: add unique id (#2483) * feat: add unique id * fix: sqlx prepare * feat: add disable option * fix: cron schedule --- backend/Cargo.lock | 2 + .../20231019154138_add_uid.down.sql | 2 + .../migrations/20231019154138_add_uid.up.sql | 2 + ...31022165924_update_hub_sync_stats.down.sql | 1 + ...0231022165924_update_hub_sync_stats.up.sql | 15 +++ backend/src/main.rs | 16 ++- backend/windmill-api/src/apps.rs | 8 +- backend/windmill-api/src/embeddings.rs | 4 +- backend/windmill-api/src/flows.rs | 8 +- backend/windmill-api/src/integration.rs | 10 +- backend/windmill-api/src/scripts.rs | 12 +- backend/windmill-common/Cargo.toml | 2 + .../windmill-common/src/global_settings.rs | 2 + backend/windmill-common/src/lib.rs | 1 + backend/windmill-common/src/scripts.rs | 23 ++-- backend/windmill-common/src/stats.rs | 110 ++++++++++++++++++ backend/windmill-common/src/utils.rs | 52 ++++++--- backend/windmill-worker/src/worker.rs | 8 +- cli/deps.ts | 4 +- cli/hub.ts | 21 +++- .../lib/components/InstanceSettings.svelte | 8 ++ 21 files changed, 246 insertions(+), 65 deletions(-) create mode 100644 backend/migrations/20231019154138_add_uid.down.sql create mode 100644 backend/migrations/20231019154138_add_uid.up.sql create mode 100644 backend/migrations/20231022165924_update_hub_sync_stats.down.sql create mode 100644 backend/migrations/20231022165924_update_hub_sync_stats.up.sql create mode 100644 backend/windmill-common/src/stats.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 89e22cadf6..b0b609dfb3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -7828,6 +7828,8 @@ dependencies = [ "anyhow", "axum", "chrono", + "cron", + "git-version", "hex", "hmac", "hyper", diff --git a/backend/migrations/20231019154138_add_uid.down.sql b/backend/migrations/20231019154138_add_uid.down.sql new file mode 100644 index 0000000000..414f99f32e --- /dev/null +++ b/backend/migrations/20231019154138_add_uid.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DELETE FROM global_settings WHERE name = 'uid'; \ No newline at end of file diff --git a/backend/migrations/20231019154138_add_uid.up.sql b/backend/migrations/20231019154138_add_uid.up.sql new file mode 100644 index 0000000000..3fecabb798 --- /dev/null +++ b/backend/migrations/20231019154138_add_uid.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +INSERT INTO global_settings (name, value, updated_at) VALUES ('uid', to_jsonb(gen_random_uuid()), now()) ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/backend/migrations/20231022165924_update_hub_sync_stats.down.sql b/backend/migrations/20231022165924_update_hub_sync_stats.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20231022165924_update_hub_sync_stats.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20231022165924_update_hub_sync_stats.up.sql b/backend/migrations/20231022165924_update_hub_sync_stats.up.sql new file mode 100644 index 0000000000..b87f066d97 --- /dev/null +++ b/backend/migrations/20231022165924_update_hub_sync_stats.up.sql @@ -0,0 +1,15 @@ +-- Add up migration script here +UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.189.0/main.ts"; +export async function main() { + await run( + "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN")); + + await run("hub", "pull"); +} + +async function run(...cmd: string[]) { + console.log("Running \"" + cmd.join('' '') + "\""); + await wmill.parse(cmd); +}', summary = 'Synchronize Hub Resource types with instance', +description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.' +WHERE hash = -28028598712388162 AND workspace_id = 'admins'; \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 3263d892e0..03272ff9cd 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -20,12 +20,14 @@ use tokio::{ fs::{metadata, DirBuilder}, sync::RwLock, }; +use windmill_api::HTTP_CLIENT; use windmill_common::{ global_settings::{ - BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, EXTRA_PIP_INDEX_URL_SETTING, - LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, + BASE_URL_SETTING, CUSTOM_TAGS_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS, + EXTRA_PIP_INDEX_URL_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, + OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, }, + stats::schedule_stats, utils::rd_string, worker::{reload_custom_tags_setting, WORKER_GROUP}, DB, METRICS_ADDR, @@ -334,7 +336,8 @@ Windmill Community Edition {GIT_VERSION} if let Err(e) = tx.send(()) { tracing::error!(error = %e, "Could not send killpill to server"); } - } + }, + DISABLE_STATS_SETTING => {}, a @_ => { tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); } @@ -378,6 +381,11 @@ Windmill Community Edition {GIT_VERSION} Ok(()) as anyhow::Result<()> }; + if mode == Mode::Server || mode == Mode::Standalone { + let instance_name = rd_string(8); + schedule_stats(&db, instance_name, &HTTP_CLIENT).await; + } + futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?; } else { tracing::info!("Nothing to do, exiting."); diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index c9a548a9a0..d87ee3cc0b 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -547,12 +547,12 @@ async fn create_app( Ok((StatusCode::CREATED, app.path)) } -async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse { +async fn list_hub_apps(Extension(db): Extension) -> impl IntoResponse { let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, "https://hub.windmill.dev/searchUiData?approved=true", - &email, None, + &db, ) .await?; Ok::<_, Error>(( @@ -563,15 +563,15 @@ async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse } pub async fn get_hub_app_by_id( - ApiAuthed { email, .. }: ApiAuthed, Path(id): Path, + Extension(db): Extension, ) -> JsonResult { let value = http_get_from_hub( &HTTP_CLIENT, &format!("https://hub.windmill.dev/apps/{id}/json"), - &email, false, None, + &db, ) .await? .json() diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index e6500b7be7..3a5d754140 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -228,9 +228,9 @@ impl EmbeddingsDb { let response = http_get_from_hub( &HTTP_CLIENT, "https://hub.windmill.dev/scripts/embeddings", - "todo@windmill.dev", false, None, + pg_db, ) .await?; let hub_scripts = response.json::>().await?; @@ -257,9 +257,9 @@ impl EmbeddingsDb { let response = http_get_from_hub( &HTTP_CLIENT, "https://hub.windmill.dev/resource_types/embeddings", - "todo@windmill.dev", false, None, + pg_db, ) .await?; let hub_resource_types = response.json::>().await?; diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 8de121aec9..58b7a2d9e5 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -163,12 +163,12 @@ async fn list_flows( Ok(Json(rows)) } -async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse { +async fn list_hub_flows(Extension(db): Extension) -> impl IntoResponse { let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, "https://hub.windmill.dev/searchFlowData?approved=true", - &email, None, + &db, ) .await?; Ok::<_, Error>(( @@ -197,15 +197,15 @@ async fn list_paths( } pub async fn get_hub_flow_by_id( - ApiAuthed { email, .. }: ApiAuthed, Path(id): Path, + Extension(db): Extension, ) -> JsonResult { let value = http_get_from_hub( &HTTP_CLIENT, &format!("https://hub.windmill.dev/flows/{id}/json"), - &email, false, None, + &db, ) .await? .json() diff --git a/backend/windmill-api/src/integration.rs b/backend/windmill-api/src/integration.rs index 1cdfc6890d..6d9c91be44 100644 --- a/backend/windmill-api/src/integration.rs +++ b/backend/windmill-api/src/integration.rs @@ -1,5 +1,7 @@ -use crate::{db::ApiAuthed, HTTP_CLIENT}; -use axum::{body::StreamBody, extract::Query, response::IntoResponse, routing::get, Router}; +use crate::{db::DB, HTTP_CLIENT}; +use axum::{ + body::StreamBody, extract::Query, response::IntoResponse, routing::get, Extension, Router, +}; use windmill_common::{error::Error, utils::query_elems_from_hub}; pub fn global_service() -> Router { @@ -11,8 +13,8 @@ struct ListHubIntegrationsQuery { kind: Option, } async fn list_hub_integrations( - ApiAuthed { email, .. }: ApiAuthed, Query(query): Query, + Extension(db): Extension, ) -> impl IntoResponse { let mut query_params = vec![]; @@ -23,8 +25,8 @@ async fn list_hub_integrations( let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, "https://hub.windmill.dev/integrations/list", - &email, Some(query_params), + &db, ) .await?; Ok::<_, Error>(( diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index eb45675804..cfb02b7680 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -259,8 +259,8 @@ struct TopHubScriptsQuery { } async fn get_top_hub_scripts( - ApiAuthed { email, .. }: ApiAuthed, Query(query): Query, + Extension(db): Extension, ) -> impl IntoResponse { let mut query_params = vec![]; if let Some(query_limit) = query.limit { @@ -276,8 +276,8 @@ async fn get_top_hub_scripts( let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, "https://hub.windmill.dev/scripts/top", - &email, Some(query_params), + &db, ) .await?; Ok::<_, Error>(( @@ -638,18 +638,18 @@ async fn create_script( } pub async fn get_hub_script_by_path( - authed: ApiAuthed, Path(path): Path, + Extension(db): Extension, ) -> Result { - windmill_common::scripts::get_hub_script_by_path(&authed.email, path, &HTTP_CLIENT).await + windmill_common::scripts::get_hub_script_by_path(path, &HTTP_CLIENT, &db).await } pub async fn get_full_hub_script_by_path( - ApiAuthed { email, .. }: ApiAuthed, Path(path): Path, + Extension(db): Extension, ) -> JsonResult { Ok(Json( - windmill_common::scripts::get_full_hub_script_by_path(&email, path, &HTTP_CLIENT).await?, + windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, &db).await?, )) } diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 1f75867d12..a90a353171 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -46,3 +46,5 @@ lazy_static.workspace = true tracing-flame = { version = "^0", optional = true } itertools.workspace = true regex.workspace = true +git-version.workspace = true +cron.workspace = true diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index c9dc3c7e9a..239eb6936f 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -7,6 +7,8 @@ pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb"; pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry"; pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; +pub const UNIQUE_ID_SETTING: &str = "uid"; +pub const DISABLE_STATS_SETTING: &str = "disable_stats"; pub const ENV_SETTINGS: [&str; 54] = [ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index b2426a700f..570f766663 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -25,6 +25,7 @@ pub mod oauth2; pub mod schedule; pub mod scripts; pub mod server; +pub mod stats; pub mod users; pub mod utils; pub mod variables; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 16f64818ef..0b48ed7445 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -11,6 +11,11 @@ use std::{ hash::{Hash, Hasher}, }; +use crate::{ + error::{to_anyhow, Error}, + utils::http_get_from_hub, + DB, +}; use serde::de::Error as _; use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; use serde_json::to_string_pretty; @@ -263,15 +268,10 @@ pub fn to_hex_string(i: &i64) -> String { #[cfg(feature = "reqwest")] pub async fn get_hub_script_by_path( - email: &str, path: StripPath, http_client: &reqwest::Client, + db: &DB, ) -> crate::error::Result { - use crate::{ - error::{to_anyhow, Error}, - utils::http_get_from_hub, - }; - let path = path .to_path() .strip_prefix("hub/") @@ -280,9 +280,9 @@ pub async fn get_hub_script_by_path( let content = http_get_from_hub( http_client, &format!("https://hub.windmill.dev/raw/{path}.ts"), - email, true, None, + db, ) .await? .text() @@ -293,15 +293,10 @@ pub async fn get_hub_script_by_path( #[cfg(feature = "reqwest")] pub async fn get_full_hub_script_by_path( - email: &str, path: StripPath, http_client: &reqwest::Client, + db: &DB, ) -> crate::error::Result { - use crate::{ - error::{to_anyhow, Error}, - utils::http_get_from_hub, - }; - let path = path .to_path() .strip_prefix("hub/") @@ -310,9 +305,9 @@ pub async fn get_full_hub_script_by_path( let value = http_get_from_hub( http_client, &format!("https://hub.windmill.dev/raw2/{path}"), - email, true, None, + db, ) .await? .json::() diff --git a/backend/windmill-common/src/stats.rs b/backend/windmill-common/src/stats.rs new file mode 100644 index 0000000000..2d7d484f9e --- /dev/null +++ b/backend/windmill-common/src/stats.rs @@ -0,0 +1,110 @@ +use std::str::FromStr; + +use crate::{ + error::{to_anyhow, Result}, + global_settings::{DISABLE_STATS_SETTING, UNIQUE_ID_SETTING}, + utils::GIT_VERSION, + DB, +}; + +use chrono::Utc; +use cron::Schedule; + +pub async fn get_disable_stats_setting(db: &DB) -> bool { + let q = sqlx::query!( + "SELECT value FROM global_settings WHERE name = $1", + DISABLE_STATS_SETTING + ) + .fetch_optional(db) + .await; + + if let Ok(q) = q { + if let Some(q) = q { + if let Ok(v) = serde_json::from_value::(q.value.clone()) { + return v; + } else { + tracing::error!( + "Could not parse DISABLE_STATS_SETTING found: {:#?}", + &q.value + ); + } + } + }; + + false +} + +pub async fn schedule_stats(db: &DB, instance_name: String, http_client: &reqwest::Client) -> () { + let http_client = http_client.clone(); + let db = db.clone(); + tokio::spawn(async move { + loop { + let disabled = get_disable_stats_setting(&db).await; + if !disabled { + tracing::info!("Sending stats"); + let result = send_stats(&instance_name, &http_client, &db).await; + if result.is_err() { + tracing::error!("Error sending stats: {}", result.err().unwrap()); + } else { + tracing::info!("Stats sent"); + } + } + + let s = "0 0 */24 * * * *"; // Every 24 hours + let s = Schedule::from_str(&s); + if s.is_err() { + tracing::error!("Invalid schedule for stats"); + return; + } + let s = s.unwrap(); + + let next_time = s.upcoming(Utc).next(); + if next_time.is_none() { + tracing::error!("Invalid schedule for stats"); + return; + } + let next_time = next_time.unwrap(); + let duration_to_next = next_time - Utc::now(); + + tokio::time::sleep(tokio::time::Duration::from_millis( + duration_to_next.num_milliseconds() as u64, + )) + .await; + } + }); +} + +pub async fn send_stats( + instance_name: &String, + http_client: &reqwest::Client, + db: &DB, +) -> Result<()> { + let uid = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + UNIQUE_ID_SETTING + ) + .fetch_one(db) + .await?; + + let uid = serde_json::from_value::(uid).map_err(to_anyhow)?; + + let payload = serde_json::json!({ + "uid": uid, + "version": GIT_VERSION, + "instance_name": instance_name, + }); + + let request = http_client + .post("https://hub.windmill.dev/stats") + .body(serde_json::to_string(&payload).map_err(to_anyhow)?) + .header("content-type", "application/json"); + + request + .send() + .await + .map_err(to_anyhow)? + .error_for_status() + .map_err(to_anyhow)?; + + Ok(()) +} diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 36660ce337..e81060613e 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -6,15 +6,22 @@ * LICENSE-AGPL for a copy of the license. */ -use crate::error::{Error, Result}; +use crate::error::{to_anyhow, Error, Result}; +use crate::global_settings::UNIQUE_ID_SETTING; +use crate::DB; +use git_version::git_version; use hyper::{HeaderMap, StatusCode}; use rand::{distributions::Alphanumeric, thread_rng, Rng}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use sqlx::{Pool, Postgres}; pub const MAX_PER_PAGE: usize = 10000; pub const DEFAULT_PER_PAGE: usize = 1000; +pub const GIT_VERSION: &str = + git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); + #[derive(Deserialize)] pub struct Pagination { pub page: Option, @@ -78,10 +85,10 @@ pub fn not_found_if_none>(opt: Option, kind: &str, name: U) pub async fn query_elems_from_hub( http_client: &reqwest::Client, url: &str, - email: &str, query_params: Option>, + db: &DB, ) -> Result<(StatusCode, HeaderMap, reqwest::Response)> { - let response = http_get_from_hub(http_client, url, email, false, query_params).await?; + let response = http_get_from_hub(http_client, url, false, query_params, db).await?; let status = response.status(); @@ -92,21 +99,34 @@ pub async fn query_elems_from_hub( pub async fn http_get_from_hub( http_client: &reqwest::Client, url: &str, - email: &str, plain: bool, query_params: Option>, + db: &Pool, ) -> Result { - let mut request = http_client - .get(url) - .header( - "Accept", - if plain { - "text/plain" - } else { - "application/json" - }, - ) - .header("X-email", email); + let uid = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + UNIQUE_ID_SETTING + ) + .fetch_optional(db) + .await? + .map(|v| serde_json::from_value::(v)); + + let mut request = http_client.get(url).header( + "Accept", + if plain { + "text/plain" + } else { + "application/json" + }, + ); + + if let Some(uid) = uid { + if let Ok(uid) = uid { + request = request.header("X-uid", uid); + } else { + tracing::info!("Invalid uid in global settings: {}", uid.err().unwrap()) + } + } if let Some(query_params) = query_params { for (key, value) in query_params { @@ -114,7 +134,7 @@ pub async fn http_get_from_hub( } } - let response = request.send().await.map_err(crate::error::to_anyhow)?; + let response = request.send().await.map_err(to_anyhow)?; Ok(response) } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ff0d1fc6c2..e8dbc437b7 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1491,9 +1491,9 @@ pub async fn process_completed_job r.json()) .then((list: { id: number; name: string }[]) => diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index eae91a8d07..48cc3bef8e 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -127,6 +127,14 @@ fieldType: 'boolean', storage: 'config' } + ], + Telemetry: [ + { + label: 'Disable telemetry', + key: 'disable_stats', + fieldType: 'boolean', + storage: 'setting' + } ] }