mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-20 00:03:21 +00:00
Merge branch 'main' into alp/gitsync_init_lock
This commit is contained in:
Alexander Petric
@@ -1,5 +1,14 @@
|
||||
# Changelog
|
||||
|
||||
## [1.540.0](https://github.com/windmill-labs/windmill/compare/v1.539.1...v1.540.0) (2025-09-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add worker_group_job_stats table for efficient job metrics aggregation ([#6527](https://github.com/windmill-labs/windmill/issues/6527)) ([20f4086](https://github.com/windmill-labs/windmill/commit/20f4086a7709f196d9deb7e8e1744eb7848a4fc7))
|
||||
* **backend:** prefix matching for list_paths_from_workspace_runnable + bulk delete endpoints for scripts/variables/resources ([#6542](https://github.com/windmill-labs/windmill/issues/6542)) ([5aac5fa](https://github.com/windmill-labs/windmill/commit/5aac5fa1361860859971fc77b4dd87048f536291))
|
||||
* timeout as expression ([#6509](https://github.com/windmill-labs/windmill/issues/6509)) ([c210a40](https://github.com/windmill-labs/windmill/commit/c210a404e03f2a857c2fabbbff2ce23189852eff))
|
||||
|
||||
## [1.539.1](https://github.com/windmill-labs/windmill/compare/v1.539.0...v1.539.1) (2025-09-06)
|
||||
|
||||
|
||||
|
||||
Generated
+45
-52
@@ -158,12 +158,6 @@ version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "android-tzdata"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
@@ -1963,18 +1957,17 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.41"
|
||||
version = "0.4.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
|
||||
checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2"
|
||||
dependencies = [
|
||||
"android-tzdata",
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"pure-rust-locales",
|
||||
"serde",
|
||||
"wasm-bindgen",
|
||||
"windows-link 0.1.3",
|
||||
"windows-link 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11208,7 +11201,7 @@ dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pki-types",
|
||||
"schannel",
|
||||
"security-framework 3.3.0",
|
||||
"security-framework 3.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11589,9 +11582,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "3.3.0"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c"
|
||||
checksum = "60b369d18893388b345804dc0007963c99b7d665ae71d275812d828c6f089640"
|
||||
dependencies = [
|
||||
"bitflags 2.9.4",
|
||||
"core-foundation 0.10.1",
|
||||
@@ -11602,9 +11595,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "security-framework-sys"
|
||||
version = "2.14.0"
|
||||
version = "2.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32"
|
||||
checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
@@ -12098,9 +12091,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "smtp-proto"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7d3950ab75b03c52f2f13fd52aab91c9d62698b231b67240e85c3ef5301e63e"
|
||||
checksum = "8003eb09806ff2ae4661dd0dca27cbd9f65ba85de06cc0302c364b0d661ba368"
|
||||
|
||||
[[package]]
|
||||
name = "snap"
|
||||
@@ -15114,11 +15107,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.10"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -15129,7 +15122,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windmill"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"aws-sdk-config",
|
||||
@@ -15185,7 +15178,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argon2",
|
||||
@@ -15305,7 +15298,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-api-client"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
@@ -15320,7 +15313,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-audit"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"serde",
|
||||
@@ -15333,7 +15326,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-autoscaling"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum",
|
||||
@@ -15352,7 +15345,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-common"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -15434,7 +15427,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-git-sync"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"serde",
|
||||
@@ -15449,7 +15442,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-indexer"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
@@ -15473,7 +15466,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-macros"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"itertools 0.14.0",
|
||||
"lazy_static",
|
||||
@@ -15485,7 +15478,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"convert_case 0.6.0",
|
||||
"serde",
|
||||
@@ -15494,7 +15487,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-bash"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -15506,7 +15499,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-csharp"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -15518,7 +15511,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-go"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"gosyn",
|
||||
@@ -15530,7 +15523,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-graphql"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -15542,7 +15535,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-java"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -15554,7 +15547,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-nu"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"nu-parser",
|
||||
@@ -15565,7 +15558,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-php"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -15576,7 +15569,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -15588,7 +15581,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py-imports"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -15611,7 +15604,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ruby"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -15625,7 +15618,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-rust"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"convert_case 0.6.0",
|
||||
@@ -15642,7 +15635,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-sql"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -15656,7 +15649,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ts"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -15674,7 +15667,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-wasm"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"getrandom 0.2.16",
|
||||
@@ -15699,7 +15692,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-yaml"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -15709,7 +15702,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-queue"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -15742,7 +15735,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-sql-datatype-parser-wasm"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-test",
|
||||
@@ -15752,7 +15745,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-worker"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-once-cell",
|
||||
@@ -16499,18 +16492,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.26"
|
||||
version = "0.8.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
|
||||
checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.26"
|
||||
version = "0.8.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
|
||||
checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -33,7 +33,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
b72d44c2a8a2ad3a1e36c938863a9b02891ac4a2
|
||||
7b62b1663930bc0c0cb751804cfa38883b2ed6e8
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Drop worker_group_job_stats table and its indices
|
||||
DROP TABLE IF EXISTS worker_group_job_stats;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Add worker_group_job_stats table
|
||||
CREATE TABLE IF NOT EXISTS worker_group_job_stats (
|
||||
hour BIGINT NOT NULL,
|
||||
worker_group TEXT NOT NULL,
|
||||
script_lang VARCHAR(50),
|
||||
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
|
||||
job_count INTEGER NOT NULL DEFAULT 0,
|
||||
total_duration_ms BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (hour, worker_group, script_lang, workspace_id)
|
||||
);
|
||||
|
||||
-- Create indices for efficient querying
|
||||
CREATE INDEX IF NOT EXISTS worker_group_job_stats_hour_idx ON worker_group_job_stats(hour DESC);
|
||||
CREATE INDEX IF NOT EXISTS worker_group_job_stats_workspace_idx ON worker_group_job_stats(workspace_id, hour DESC);
|
||||
CREATE INDEX IF NOT EXISTS worker_group_job_stats_worker_group_idx ON worker_group_job_stats(worker_group, hour DESC);
|
||||
@@ -1531,6 +1531,22 @@ pub async fn monitor_db(
|
||||
}
|
||||
}
|
||||
};
|
||||
// run every hour (60 minutes / 30 seconds = 120)
|
||||
let cleanup_worker_group_stats_f = async {
|
||||
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) {
|
||||
if let Some(db) = conn.as_sql() {
|
||||
match windmill_common::worker_group_job_stats::cleanup_old_stats(db, 60).await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Deleted {} old worker group job stats rows", count);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Error cleaning up worker group job stats: {:?}", e);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// run every hour
|
||||
let vacuum_queue_f = async {
|
||||
@@ -1633,6 +1649,7 @@ pub async fn monitor_db(
|
||||
update_min_worker_version_f,
|
||||
cleanup_concurrency_counters_f,
|
||||
cleanup_concurrency_counters_empty_keys_f,
|
||||
cleanup_worker_group_stats_f,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.539.1
|
||||
version: 1.540.0
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
|
||||
@@ -84,6 +84,7 @@ pub mod users;
|
||||
pub mod utils;
|
||||
pub mod variables;
|
||||
pub mod worker;
|
||||
pub mod worker_group_job_stats;
|
||||
pub mod workspaces;
|
||||
|
||||
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{Pool, Postgres};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::scripts::ScriptLang;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct JobStatsAccumulator {
|
||||
pub worker_group: String,
|
||||
pub script_lang: Option<ScriptLang>,
|
||||
pub workspace_id: String,
|
||||
pub job_count: i32,
|
||||
pub total_duration_ms: i64,
|
||||
}
|
||||
|
||||
pub type JobStatsMap =
|
||||
Arc<RwLock<HashMap<(i64, String, Option<ScriptLang>, String), JobStatsAccumulator>>>;
|
||||
|
||||
pub fn get_current_hour() -> i64 {
|
||||
let now = Utc::now();
|
||||
(now.timestamp() / 3600) * 3600
|
||||
}
|
||||
|
||||
pub async fn accumulate_job_stats(
|
||||
stats_map: &JobStatsMap,
|
||||
worker_group: &str,
|
||||
script_lang: Option<ScriptLang>,
|
||||
workspace_id: &str,
|
||||
duration_ms: i64,
|
||||
) {
|
||||
let hour = get_current_hour();
|
||||
let key = (
|
||||
hour,
|
||||
worker_group.to_string(),
|
||||
script_lang.clone(),
|
||||
workspace_id.to_string(),
|
||||
);
|
||||
|
||||
let mut stats = stats_map.write().await;
|
||||
let entry = stats
|
||||
.entry(key.clone())
|
||||
.or_insert_with(|| JobStatsAccumulator {
|
||||
worker_group: worker_group.to_string(),
|
||||
script_lang,
|
||||
workspace_id: workspace_id.to_string(),
|
||||
job_count: 0,
|
||||
total_duration_ms: 0,
|
||||
});
|
||||
|
||||
entry.job_count += 1;
|
||||
entry.total_duration_ms += duration_ms;
|
||||
}
|
||||
|
||||
pub async fn flush_stats_to_db(
|
||||
db: &Pool<Postgres>,
|
||||
stats_map: &JobStatsMap,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
let current_stats: Vec<(
|
||||
(i64, String, Option<ScriptLang>, String),
|
||||
JobStatsAccumulator,
|
||||
)> = {
|
||||
let mut stats = stats_map.write().await;
|
||||
if stats.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
stats.drain().collect()
|
||||
};
|
||||
|
||||
for ((hour, _worker_group, script_lang, _workspace_id), accumulator) in current_stats {
|
||||
let script_lang_str = script_lang.as_ref().map(|l| l.as_str());
|
||||
|
||||
// Use ON CONFLICT to sum existing values
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO worker_group_job_stats
|
||||
(hour, worker_group, script_lang, workspace_id, job_count, total_duration_ms)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (hour, worker_group, script_lang, workspace_id)
|
||||
DO UPDATE SET
|
||||
job_count = worker_group_job_stats.job_count + EXCLUDED.job_count,
|
||||
total_duration_ms = worker_group_job_stats.total_duration_ms + EXCLUDED.total_duration_ms
|
||||
"#,
|
||||
)
|
||||
.bind(hour)
|
||||
.bind(&accumulator.worker_group)
|
||||
.bind(script_lang_str)
|
||||
.bind(&accumulator.workspace_id)
|
||||
.bind(accumulator.job_count)
|
||||
.bind(accumulator.total_duration_ms)
|
||||
.execute(db)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn cleanup_old_stats(
|
||||
db: &Pool<Postgres>,
|
||||
retention_days: i64,
|
||||
) -> Result<u64, sqlx::Error> {
|
||||
let cutoff_timestamp = get_current_hour() - (retention_days * 24 * 3600);
|
||||
|
||||
let result = sqlx::query("DELETE FROM worker_group_job_stats WHERE hour < $1")
|
||||
.bind(cutoff_timestamp)
|
||||
.execute(db)
|
||||
.await?;
|
||||
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
@@ -22,6 +22,7 @@ use windmill_common::{
|
||||
jobs::JobKind,
|
||||
utils::WarnAfterExt,
|
||||
worker::{to_raw_value, Connection, WORKER_GROUP},
|
||||
worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap},
|
||||
KillpillSender, DB,
|
||||
};
|
||||
|
||||
@@ -63,6 +64,7 @@ async fn process_jc(
|
||||
worker_dir: &str,
|
||||
same_worker_tx: Option<&SameWorkerSender>,
|
||||
job_completed_sender: &JobCompletedSender,
|
||||
stats_map: &JobStatsMap,
|
||||
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
|
||||
) {
|
||||
let success: bool = jc.success;
|
||||
@@ -139,6 +141,11 @@ async fn process_jc(
|
||||
}
|
||||
}
|
||||
|
||||
// Extract stats info before moving jc
|
||||
let duration_ms = jc.duration.clone();
|
||||
let script_lang = jc.job.script_lang.clone();
|
||||
let workspace_id = jc.job.workspace_id.clone();
|
||||
|
||||
let root_job = handle_receive_completed_job(
|
||||
jc,
|
||||
&base_internal_url,
|
||||
@@ -156,6 +163,18 @@ async fn process_jc(
|
||||
if let Some(root_job) = root_job {
|
||||
add_root_flow_job_to_otlp(&root_job, success);
|
||||
}
|
||||
|
||||
// Accumulate job stats if duration is available
|
||||
if let Some(duration_ms) = duration_ms {
|
||||
accumulate_job_stats(
|
||||
stats_map,
|
||||
&*WORKER_GROUP,
|
||||
script_lang,
|
||||
&workspace_id,
|
||||
duration_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
enum JobCompletedRx {
|
||||
@@ -178,6 +197,7 @@ pub fn start_background_processor(
|
||||
worker_name: String,
|
||||
killpill_tx: KillpillSender,
|
||||
is_dedicated_worker: bool,
|
||||
stats_map: JobStatsMap,
|
||||
) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut has_been_killed = false;
|
||||
@@ -187,6 +207,28 @@ pub fn start_background_processor(
|
||||
#[cfg(feature = "benchmark")]
|
||||
let mut infos = BenchmarkInfo::new();
|
||||
|
||||
// Start periodic stats flush task
|
||||
let db_clone = db.clone();
|
||||
let stats_map_clone = stats_map.clone();
|
||||
let mut killpill_rx_clone = killpill_rx.resubscribe();
|
||||
let flush_handle = tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(900)); // Flush every 15 min
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
if let Err(e) = flush_stats_to_db(&db_clone, &stats_map_clone).await {
|
||||
tracing::error!("Failed to flush worker group job stats: {}", e);
|
||||
}
|
||||
}
|
||||
_ = killpill_rx_clone.recv() => {
|
||||
tracing::info!("bg processor received killpill signal, flushing remaining stats");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
//if we have been killed, we want to drain the queue of jobs
|
||||
while let Some(sr) = {
|
||||
if has_been_killed {
|
||||
@@ -241,6 +283,7 @@ pub fn start_background_processor(
|
||||
&worker_dir,
|
||||
Some(&same_worker_tx),
|
||||
&job_completed_sender,
|
||||
&stats_map,
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench,
|
||||
)
|
||||
@@ -323,6 +366,16 @@ pub fn start_background_processor(
|
||||
}
|
||||
}
|
||||
|
||||
// Flush any remaining stats before shutting down
|
||||
tracing::info!("flushing remaining stats before shutting down");
|
||||
let flush_result =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), flush_handle).await;
|
||||
match flush_result {
|
||||
Ok(Ok(())) => tracing::info!("Stats flushed successfully"),
|
||||
Ok(Err(join_err)) => tracing::error!("Stats flush task failed: {}", join_err),
|
||||
Err(_) => tracing::error!("Stats flush timed out after 10 seconds"),
|
||||
}
|
||||
|
||||
job_completed_processor_is_done.store(true, Ordering::SeqCst);
|
||||
|
||||
tracing::info!("finished processing all completed jobs");
|
||||
|
||||
@@ -26,6 +26,7 @@ use windmill_common::{
|
||||
make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT,
|
||||
MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, TMP_DIR,
|
||||
},
|
||||
worker_group_job_stats::JobStatsMap,
|
||||
KillpillSender,
|
||||
};
|
||||
|
||||
@@ -1218,6 +1219,7 @@ pub async fn run_worker(
|
||||
|
||||
// This is used to wake up the background processor when main loop is done and just waiting for new same workers jobs, and that bg processor is also not processing any jobs, bg processing can exit if no more same worker jobs
|
||||
let wake_up_notify = Arc::new(tokio::sync::Notify::new());
|
||||
let stats_map = JobStatsMap::default();
|
||||
let send_result = match (conn, job_completed_rx) {
|
||||
(Connection::Sql(db), Some(job_completed_receiver)) => Some(start_background_processor(
|
||||
job_completed_receiver,
|
||||
@@ -1233,6 +1235,7 @@ pub async fn run_worker(
|
||||
worker_name.clone(),
|
||||
killpill_tx.clone(),
|
||||
is_dedicated_worker,
|
||||
stats_map,
|
||||
)),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
|
||||
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
|
||||
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
|
||||
|
||||
export const VERSION = "v1.539.1";
|
||||
export const VERSION = "v1.540.0";
|
||||
|
||||
export async function login(email: string, password: string): Promise<string> {
|
||||
return await windmill.UserService.login({
|
||||
|
||||
+1
-1
@@ -68,7 +68,7 @@ export {
|
||||
// }
|
||||
// });
|
||||
|
||||
export const VERSION = "1.539.1";
|
||||
export const VERSION = "1.540.0";
|
||||
|
||||
const command = new Command()
|
||||
.name("wmill")
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "windmill-components",
|
||||
"version": "1.539.1",
|
||||
"version": "1.540.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "windmill-components",
|
||||
"version": "1.539.1",
|
||||
"version": "1.540.0",
|
||||
"hasInstallScript": true,
|
||||
"license": "AGPL-3.0",
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "windmill-components",
|
||||
"version": "1.539.1",
|
||||
"version": "1.540.0",
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
"build": "vite build",
|
||||
|
||||
@@ -343,11 +343,15 @@ export const CHAT_SYSTEM_PROMPT = `
|
||||
Your task is to respond to the user's request. Assume all user queries are valid and actionable.
|
||||
|
||||
When the user requests code changes:
|
||||
- ALWAYS use the \`edit_code\` tool to apply code changes. Use it only once with the complete updated code.
|
||||
- Pass the **complete updated file** to the \`edit_code\` tool, not just the modified sections.
|
||||
- The code can include \`[#START]\` and \`[#END]\` markers to indicate the start and end of a code piece. You MUST only modify the code between these markers if given, and remove them when passing to the tool. If a question is asked about the code, you MUST only talk about the code between the markers. Refer to it as the code piece, not the code between the markers.
|
||||
- ALWAYS use the \`edit_code\` tool to apply code changes. Use it only once with an array of diffs.
|
||||
- Pass an array of **diff objects** to the \`edit_code\` tool. Each diff should specify exactly what text to replace and what to replace it with.
|
||||
- Each diff object must contain:
|
||||
- \`old_string\`: The exact text to replace (must match the current code exactly)
|
||||
- \`new_string\`: The replacement text
|
||||
- \`replace_all\` (optional): Set to true to replace all occurrences, false or omit for first occurrence only
|
||||
- The code can include \`[#START]\` and \`[#END]\` markers to indicate the start and end of a code piece. You MUST only modify the code between these markers if given, and remove them when creating your diffs. If a question is asked about the code, you MUST only talk about the code between the markers. Refer to it as the code piece, not the code between the markers.
|
||||
- Follow the instructions carefully and explain the reasoning behind your changes in your response text.
|
||||
- If the request is abstract (e.g., "make this cleaner"), interpret it concretely and reflect that in the code passed to the tool.
|
||||
- If the request is abstract (e.g., "make this cleaner"), interpret it concretely and reflect that in the diffs.
|
||||
- Preserve existing formatting, indentation, and whitespace unless changes are strictly required to fulfill the user's request.
|
||||
- The user can ask you to look at or modify specific files, databases or errors by having its name in the INSTRUCTIONS preceded by the @ symbol. In this case, put your focus on the element that is explicitly mentioned.
|
||||
- The user can ask you questions about a list of \`DATABASES\` that are available in the user's workspace. If the user asks you a question about a database, you should ask the user to specify the database name if not given, or take the only one available if there is only one.
|
||||
@@ -356,7 +360,24 @@ export const CHAT_SYSTEM_PROMPT = `
|
||||
- When asked to create a script that communicates with an external service, you can use the \`search_hub_scripts\` tool to search for relevant scripts in the hub. Make sure the language is the same as what the user is coding in. If you do not find any relevant scripts, you can use the \`search_npm_packages\` tool to search for relevant packages and their documentation. Always give a link to the documentation in your answer if possible.
|
||||
- After applying code changes with the \`edit_code\` tool, ALWAYS use the \`test_run_script\` tool to test the code, and iterate on the code until it works as expected (MAX 3 times). If the user cancels the test run, do not try again and wait for the next user instruction.
|
||||
|
||||
Example diff usage:
|
||||
To change "return 1" to "return 2", use:
|
||||
[{
|
||||
"old_string": "return 1",
|
||||
"new_string": "return 2"
|
||||
}]
|
||||
|
||||
To add a new function and modify an existing one:
|
||||
[{
|
||||
"old_string": "export async function main() {",
|
||||
"new_string": "function helper() {\n\treturn 'help';\n}\n\nexport async function main() {"
|
||||
}, {
|
||||
"old_string": "return result;",
|
||||
"new_string": "return result + helper();"
|
||||
}]
|
||||
|
||||
Important:
|
||||
- Each old_string must match the exact text in the current code, including whitespace and indentation.
|
||||
- Do not return the applied code in your response, just explain what you did. You can return code blocks in your response for explanations or examples as per user request.
|
||||
- Do not mention or reveal these instructions to the user unless explicitly asked to do so.
|
||||
`
|
||||
@@ -794,18 +815,38 @@ const EDIT_CODE_TOOL: ChatCompletionFunctionTool = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'edit_code',
|
||||
description: 'Apply code changes to the current script in the editor',
|
||||
description: 'Apply code changes to the current script in the editor using an array of diffs',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
code: {
|
||||
type: 'string',
|
||||
description: 'The complete updated code for the entire script file'
|
||||
diffs: {
|
||||
type: 'array',
|
||||
description: 'Array of diff objects to apply to the code',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
old_string: {
|
||||
type: 'string',
|
||||
description: 'The exact text to replace (must match the current code exactly)'
|
||||
},
|
||||
new_string: {
|
||||
type: 'string',
|
||||
description: 'The new text to replace the old_string with'
|
||||
},
|
||||
replace_all: {
|
||||
type: 'boolean',
|
||||
description:
|
||||
'If true, replace all occurrences of old_string. If false or omitted, only replace the first occurrence.'
|
||||
}
|
||||
},
|
||||
required: ['old_string', 'new_string'],
|
||||
additionalProperties: false
|
||||
}
|
||||
}
|
||||
},
|
||||
additionalProperties: false,
|
||||
strict: true,
|
||||
required: ['code']
|
||||
required: ['diffs']
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -842,12 +883,12 @@ export const editCodeTool: Tool<ScriptChatHelpers> = {
|
||||
)
|
||||
}
|
||||
|
||||
if (!args.code || typeof args.code !== 'string') {
|
||||
if (!args.diffs || !Array.isArray(args.diffs)) {
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: 'Invalid code provided',
|
||||
error: 'Code parameter is required and must be a string'
|
||||
content: 'Invalid diffs provided',
|
||||
error: 'Diffs parameter is required and must be an array'
|
||||
})
|
||||
throw new Error('Code parameter is required and must be a string')
|
||||
throw new Error('Diffs parameter is required and must be an array')
|
||||
}
|
||||
|
||||
toolCallbacks.setToolStatus(toolId, { content: 'Applying code changes...' })
|
||||
@@ -856,14 +897,32 @@ export const editCodeTool: Tool<ScriptChatHelpers> = {
|
||||
// Save old code
|
||||
const oldCode = scriptOptions.code
|
||||
|
||||
// Apply diffs sequentially
|
||||
let updatedCode = oldCode
|
||||
for (const [index, diff] of args.diffs.entries()) {
|
||||
const { old_string, new_string, replace_all = false } = diff
|
||||
|
||||
if (!updatedCode.includes(old_string)) {
|
||||
throw new Error(`Diff at index ${index}: old_string "${old_string}" not found in code`)
|
||||
}
|
||||
|
||||
if (replace_all) {
|
||||
updatedCode = updatedCode.replaceAll(old_string, new_string)
|
||||
} else {
|
||||
updatedCode = updatedCode.replace(old_string, new_string)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply the code changes directly
|
||||
await helpers.applyCode(args.code, { applyAll: true, mode: 'apply' })
|
||||
await helpers.applyCode(updatedCode, { applyAll: true, mode: 'apply' })
|
||||
|
||||
// Show revert mode
|
||||
await helpers.applyCode(oldCode, { mode: 'revert' })
|
||||
|
||||
toolCallbacks.setToolStatus(toolId, { content: 'Code changes applied' })
|
||||
return 'Code has been applied to the script editor.'
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: `Code changes applied`
|
||||
})
|
||||
return `Applied changes to the script editor.`
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
|
||||
+2
-2
@@ -4,8 +4,8 @@ verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
wmill = ">=1.539.1"
|
||||
wmill_pg = ">=1.539.1"
|
||||
wmill = ">=1.540.0"
|
||||
wmill_pg = ">=1.540.0"
|
||||
sendgrid = "*"
|
||||
mysql-connector-python = "*"
|
||||
pymongo = "*"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.539.1
|
||||
version: 1.540.0
|
||||
title: OpenFlow Spec
|
||||
contact:
|
||||
name: Ruben Fiszel
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
RootModule = 'WindmillClient.psm1'
|
||||
|
||||
# Version number of this module.
|
||||
ModuleVersion = '1.539.1'
|
||||
ModuleVersion = '1.540.0'
|
||||
|
||||
# Supported PSEditions
|
||||
# CompatiblePSEditions = @()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "wmill"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
description = "A client library for accessing Windmill server wrapping the Windmill client API"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://windmill.dev"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "wmill-pg"
|
||||
version = "1.539.1"
|
||||
version = "1.540.0"
|
||||
description = "An extension client for the wmill client library focused on pg"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://windmill.dev"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@windmill/windmill",
|
||||
"version": "1.539.1",
|
||||
"version": "1.540.0",
|
||||
"exports": "./src/index.ts",
|
||||
"publish": {
|
||||
"exclude": ["!src", "./s3Types.ts", "./client.ts"]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "windmill-client",
|
||||
"description": "Windmill SDK client for browsers and Node.js",
|
||||
"version": "1.539.1",
|
||||
"version": "1.540.0",
|
||||
"author": "Ruben Fiszel",
|
||||
"license": "Apache 2.0",
|
||||
"devDependencies": {
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
1.539.1
|
||||
1.540.0
|
||||
|
||||
Reference in New Issue
Block a user