diff --git a/backend/.sqlx/query-db70c34599169bac0c5c2cc0e3d305f1f95c63e566a9ecacd76dc3ada4b34aee.json b/backend/.sqlx/query-2e72e6d37c3698f60d1f7b71f2bd48cd09a6a1805d6b0a63d743c18d7d2bf36c.json similarity index 78% rename from backend/.sqlx/query-db70c34599169bac0c5c2cc0e3d305f1f95c63e566a9ecacd76dc3ada4b34aee.json rename to backend/.sqlx/query-2e72e6d37c3698f60d1f7b71f2bd48cd09a6a1805d6b0a63d743c18d7d2bf36c.json index af1e1a217a..11bf3121b8 100644 --- a/backend/.sqlx/query-db70c34599169bac0c5c2cc0e3d305f1f95c63e566a9ecacd76dc3ada4b34aee.json +++ b/backend/.sqlx/query-2e72e6d37c3698f60d1f7b71f2bd48cd09a6a1805d6b0a63d743c18d7d2bf36c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.created_at > $1 AND cj.created_at < $3)\n OR cj.id = ANY($2)\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4\n ORDER BY created_at ASC", + "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.created_at > $1 AND cj.created_at < $3)\n OR cj.id = ANY($2)\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4 OR row_num = 1\n ORDER BY created_at ASC", "describe": { "columns": [ { @@ -245,5 +245,5 @@ true ] }, - "hash": "db70c34599169bac0c5c2cc0e3d305f1f95c63e566a9ecacd76dc3ada4b34aee" + "hash": "2e72e6d37c3698f60d1f7b71f2bd48cd09a6a1805d6b0a63d743c18d7d2bf36c" } diff --git a/backend/.sqlx/query-2becad3d3ebea1cbc9e9ce7c641adea5f6476ec0f83dcce2fcbde39652a64105.json b/backend/.sqlx/query-43f4555c7bf42b5d844303ccc32d84e37f27ffb39ea6f2936eb5ee9bf1c130fd.json similarity index 78% rename from backend/.sqlx/query-2becad3d3ebea1cbc9e9ce7c641adea5f6476ec0f83dcce2fcbde39652a64105.json rename to backend/.sqlx/query-43f4555c7bf42b5d844303ccc32d84e37f27ffb39ea6f2936eb5ee9bf1c130fd.json index 0701087678..7f1fb1281a 100644 --- a/backend/.sqlx/query-2becad3d3ebea1cbc9e9ce7c641adea5f6476ec0f83dcce2fcbde39652a64105.json +++ b/backend/.sqlx/query-43f4555c7bf42b5d844303ccc32d84e37f27ffb39ea6f2936eb5ee9bf1c130fd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.created_at < $1\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2\n ORDER BY created_at ASC", + "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n cj.parent_job,\n cj.created_by,\n cj.duration_ms,\n cj.success,\n cj.script_hash,\n cj.script_path,\n cj.args,\n cj.result,\n cj.deleted,\n cj.canceled,\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind,\n cj.schedule_path,\n cj.permissioned_as,\n cj.is_flow_step,\n cj.language,\n cj.is_skipped,\n cj.email,\n cj.visible_to_owner,\n cj.mem_peak,\n cj.tag,\n cj.created_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(cj.created_by), 0) + \n COALESCE(LENGTH(cj.script_path), 0) + \n COALESCE(LENGTH(cj.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.created_at < $1\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY created_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY created_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n created_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2 OR row_num = 1\n ORDER BY created_at ASC", "describe": { "columns": [ { @@ -243,5 +243,5 @@ true ] }, - "hash": "2becad3d3ebea1cbc9e9ce7c641adea5f6476ec0f83dcce2fcbde39652a64105" + "hash": "43f4555c7bf42b5d844303ccc32d84e37f27ffb39ea6f2936eb5ee9bf1c130fd" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 81dc1433b4..a648116f35 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -748e285b407e5fa7b18e04a3aa4522e42f6ff03b \ No newline at end of file +507f353e2e3af79818230c36210e86bb33c8b088 \ No newline at end of file diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 1a24028fc3..e581196af8 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -21,7 +21,7 @@ impl Default for TantivyIndexerSettings { fn default() -> Self { TantivyIndexerSettings { writer_memory_budget: 300_000_000, - commit_job_max_batch_size: 100_000, + commit_job_max_batch_size: 50_000, commit_log_max_batch_size: 10_000, refresh_index_period: 300, refresh_log_index_period: 300,