starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-11 00:06:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

indexer improvements

Browse Source
This commit is contained in:
Ruben Fiszel
2025-07-31 07:40:54 +00:00
parent d71811d384
commit d81e190cf9
4 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-db70c34599169bac0c5c2cc0e3d305f1f95c63e566a9ecacd76dc3ada4b34aee.json → backend/.sqlx/query-2e72e6d37c3698f60d1f7b71f2bd48cd09a6a1805d6b0a63d743c18d7d2bf36c.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.created_at > $1 AND cj.created_at < $3)\n OR cj.id = ANY($2)\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option<ScriptHash>\",\n script_path,\n args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4\n ORDER BY created_at ASC",
"query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.created_at > $1 AND cj.created_at < $3)\n OR cj.id = ANY($2)\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option<ScriptHash>\",\n script_path,\n args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4 OR row_num = 1\n ORDER BY created_at ASC",
"describe": {
"columns": [
{
@@ -245,5 +245,5 @@
true
]
},
"hash": "db70c34599169bac0c5c2cc0e3d305f1f95c63e566a9ecacd76dc3ada4b34aee"
"hash": "2e72e6d37c3698f60d1f7b71f2bd48cd09a6a1805d6b0a63d743c18d7d2bf36c"
}
backend/.sqlx/query-2becad3d3ebea1cbc9e9ce7c641adea5f6476ec0f83dcce2fcbde39652a64105.json → backend/.sqlx/query-43f4555c7bf42b5d844303ccc32d84e37f27ffb39ea6f2936eb5ee9bf1c130fd.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.created_at < $1\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option<ScriptHash>\",\n script_path,\n args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2\n ORDER BY created_at ASC",
"query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.created_at < $1\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option<ScriptHash>\",\n script_path,\n args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2 OR row_num = 1\n ORDER BY created_at ASC",
"describe": {
"columns": [
{
@@ -243,5 +243,5 @@
true
]
},
"hash": "2becad3d3ebea1cbc9e9ce7c641adea5f6476ec0f83dcce2fcbde39652a64105"
"hash": "43f4555c7bf42b5d844303ccc32d84e37f27ffb39ea6f2936eb5ee9bf1c130fd"
}
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
748e285b407e5fa7b18e04a3aa4522e42f6ff03b
507f353e2e3af79818230c36210e86bb33c8b088
backend/windmill-common/src/indexer.rs
+1 -1
View File
@@ -21,7 +21,7 @@ impl Default for TantivyIndexerSettings {
fn default() -> Self {
TantivyIndexerSettings {
writer_memory_budget: 300_000_000,
commit_job_max_batch_size: 100_000,
commit_job_max_batch_size: 50_000,
commit_log_max_batch_size: 10_000,
refresh_index_period: 300,
refresh_log_index_period: 300,