diff --git a/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json new file mode 100644 index 0000000000..708922a0cb --- /dev/null +++ b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, storage, inflight_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, upload_id)\n DO UPDATE SET inflight_bytes = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f" +} diff --git a/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json new file mode 100644 index 0000000000..16063b9a3f --- /dev/null +++ b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7" +} diff --git a/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json new file mode 100644 index 0000000000..291dc00310 --- /dev/null +++ b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2 AND created_at > now() - ($3::text)::interval)::bigint as \"other_reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b" +} diff --git a/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json new file mode 100644 index 0000000000..137af7638b --- /dev/null +++ b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129" +} diff --git a/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json new file mode 100644 index 0000000000..18af751ca0 --- /dev/null +++ b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('workspace_multipart_inflight'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc" +} diff --git a/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json new file mode 100644 index 0000000000..cd2dd23319 --- /dev/null +++ b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3))::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce" +} diff --git a/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json new file mode 100644 index 0000000000..be54b4a25d --- /dev/null +++ b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2" +} diff --git a/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json new file mode 100644 index 0000000000..7250aff598 --- /dev/null +++ b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at < now() - ($2::text)::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f" +} diff --git a/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json new file mode 100644 index 0000000000..c0207b6bc7 --- /dev/null +++ b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, part_id, storage, part_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, upload_id, part_id)\n DO UPDATE SET part_bytes = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a" +} diff --git a/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json new file mode 100644 index 0000000000..253fe79753 --- /dev/null +++ b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n -- reservation of every OTHER in-flight upload\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2\n AND created_at > now() - ($4::text)::interval\n GROUP BY upload_id) t)::bigint as \"other_reserved!\",\n -- this upload's already-recorded parts, excluding the part being (re)uploaded\n (SELECT COALESCE(SUM(part_bytes), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 AND part_id <> $3)::bigint as \"this_other_parts!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "this_other_parts!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c" +} diff --git a/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json new file mode 100644 index 0000000000..0c3fdb8d0a --- /dev/null +++ b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_xact_lock(hashtext('workspace_storage_usage'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_xact_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03" +} diff --git a/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json new file mode 100644 index 0000000000..691016a610 --- /dev/null +++ b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') as \"key!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d" +} diff --git a/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json new file mode 100644 index 0000000000..0f1bcc2271 --- /dev/null +++ b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT inflight_bytes, target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "inflight_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f" +} diff --git a/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json new file mode 100644 index 0000000000..1148679230 --- /dev/null +++ b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(SUM(bytes), 0)::bigint as \"total!\",\n COALESCE(MIN(computed_at) < now() - interval '10 minutes', true) as \"stale!\"\n FROM workspace_storage_usage WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683" +} diff --git a/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json new file mode 100644 index 0000000000..96608e1bc2 --- /dev/null +++ b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, GREATEST($3::bigint, 0), to_timestamp(0))\n ON CONFLICT (workspace_id, storage)\n DO UPDATE SET bytes = GREATEST(workspace_storage_usage.bytes + $3::bigint, 0)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616" +} diff --git a/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json new file mode 100644 index 0000000000..abb128f463 --- /dev/null +++ b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT storage, bytes, computed_at FROM workspace_storage_usage\n WHERE workspace_id = $1 ORDER BY storage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "storage", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "computed_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b" +} diff --git a/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json new file mode 100644 index 0000000000..1fd1fbaece --- /dev/null +++ b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_storage_usage WHERE workspace_id = $1 AND storage != ALL($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a" +} diff --git a/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json new file mode 100644 index 0000000000..5609adaad8 --- /dev/null +++ b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3)\n GROUP BY upload_id) t)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c" +} diff --git a/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json new file mode 100644 index 0000000000..4f0731f5e4 --- /dev/null +++ b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, $3, now())\n ON CONFLICT (workspace_id, storage) DO UPDATE SET bytes = $3, computed_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5e76f395d2..45adcf5f9f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -c3852ecb36bd0be1a74c63169e513888f3347850 +bea5a8b5120d6d69cab1ad4611ebe463559bd200 diff --git a/backend/migrations/20260702064737_workspace_storage_usage.down.sql b/backend/migrations/20260702064737_workspace_storage_usage.down.sql new file mode 100644 index 0000000000..e32480a602 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_storage_usage; diff --git a/backend/migrations/20260702064737_workspace_storage_usage.up.sql b/backend/migrations/20260702064737_workspace_storage_usage.up.sql new file mode 100644 index 0000000000..d7274e8742 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.up.sql @@ -0,0 +1,17 @@ +-- Cached per-(workspace, storage) byte usage of workspace object storage, +-- refreshed by listing the storage location and adjusted optimistically as +-- uploads complete. Read on every workspace-storage write in CE builds to +-- enforce the storage quota, and by the storage_usage endpoint in all builds. +CREATE TABLE workspace_storage_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + storage VARCHAR(255) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + computed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, storage) +); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role +-- that set them (same gap as workspace_diff, notify_event, script_trigger). +GRANT ALL ON workspace_storage_usage TO windmill_user; +GRANT ALL ON workspace_storage_usage TO windmill_admin; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql new file mode 100644 index 0000000000..e72edd0b12 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_multipart_inflight; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql new file mode 100644 index 0000000000..9768eaf842 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql @@ -0,0 +1,32 @@ +-- Reservation for the parts of in-flight (initiated but not yet completed) +-- multipart uploads to workspace object storage. Uncommitted parts occupy +-- object-store capacity but are invisible to the list-based storage recount +-- until completion, so CE folds this reservation into the remaining quota to +-- bound abandoned uploads. One row per uploaded part so a re-uploaded part +-- (same part_id) replaces rather than double-counts; a part is recorded only +-- after its upstream upload succeeds. Rows are removed on successful complete +-- and lazily expired after a TTL (abort/abandon rely on the TTL, which matches +-- when the object store reaps the uncommitted parts). +-- part_id - S3 part number or Azure block id (string) +-- part_bytes - size of that part +-- target_existing_size - size of the object the upload will overwrite (0 if new), +-- credited so an overwrite only reserves the net growth +CREATE TABLE workspace_multipart_inflight ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + upload_id VARCHAR(512) NOT NULL, + part_id VARCHAR(256) NOT NULL, + storage VARCHAR(255) NOT NULL, + part_bytes BIGINT NOT NULL DEFAULT 0, + target_existing_size BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, upload_id, part_id) +); + +CREATE INDEX idx_workspace_multipart_inflight_created_at + ON workspace_multipart_inflight (created_at); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role that +-- set them (same gap as workspace_storage_usage, notify_event, script_trigger). +GRANT ALL ON workspace_multipart_inflight TO windmill_user; +GRANT ALL ON workspace_multipart_inflight TO windmill_admin; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index eec76ea009..e661c3aeb8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -19901,6 +19901,55 @@ paths: application/json: schema: {} + /w/{workspace}/job_helpers/storage_usage: + get: + summary: Get the storage usage of the workspace object storage, per configured storage. On Community Edition, also returns the workspace storage quota. + operationId: getStorageUsage + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: refresh + in: query + description: recount usage by listing the storage instead of returning cached values + schema: + type: boolean + responses: + "200": + description: Storage usage + content: + application/json: + schema: + type: object + properties: + total_bytes: + type: integer + format: int64 + quota_bytes: + type: integer + format: int64 + description: only present on Community Edition, where workspace storage is capped + storages: + type: array + items: + type: object + properties: + storage: + type: string + bytes: + type: integer + format: int64 + computed_at: + type: string + format: date-time + required: + - storage + - bytes + - computed_at + required: + - total_bytes + - storages + /w/{workspace}/job_helpers/list_stored_files: get: summary: List the file keys available in a workspace object storage diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 38fa5ff862..99d6156687 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -7,6 +7,11 @@ use std::{collections::HashMap, sync::Arc}; * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ +#[cfg(all(feature = "parquet", not(feature = "enterprise")))] +use crate::job_helpers_oss::{ + bump_storage_usage, ce_upload_budget, reject_reserved_volume_key, + spawn_storage_usage_recount_floored, +}; use crate::{ auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, @@ -3544,7 +3549,49 @@ async fn upload_s3_file_from_app( ]) .into(); - let _put_result = upload_file_from_req(s3_client, &file_key, request, options).await?; + // Only workspace storage is quota-metered; a custom-resource upload lands in + // the user's own bucket and is neither capped nor counted. An overwrite of an + // existing key only spends the difference over its current size. + let _is_workspace_storage = query.s3_resource_path.is_none(); + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + reject_reserved_volume_key(&file_key)?; + } + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + let (max_size, _existing_size) = if _is_workspace_storage { + let content_length = request + .headers() + .get(http::header::CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.parse::().ok()); + let budget = ce_upload_budget(&db, &w_id, &s3_client, &file_key, content_length).await?; + (Some(budget.max_size), budget.existing_size) + } else { + (None, 0) + }; + #[cfg(any(not(feature = "parquet"), feature = "enterprise"))] + let max_size: Option = None; + + match upload_file_from_req(s3_client, &file_key, request, options, max_size).await { + Ok((_, _size)) => + { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + bump_storage_usage( + &db, + &w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64 - _existing_size, + ) + .await; + } + } + Err(e) => { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + spawn_storage_usage_recount_floored(&db, &w_id); + return Err(e); + } + } let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims { file_key: file_key.clone(), diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index 3793e94b02..dc8b36fbaf 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -90,6 +90,10 @@ impl RawWebhookArgs { db: &DB, w_id: &str, ) -> Result>, Error> { + #[cfg(not(feature = "enterprise"))] + use crate::job_helpers_oss::{ + bump_storage_usage, ce_storage_quota_remaining, spawn_storage_usage_recount_floored, + }; use crate::job_helpers_oss::{ get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; @@ -139,8 +143,38 @@ impl RawWebhookArgs { .into_stream() .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)); - upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options) - .await?; + // file_key is always freshly random here, so this never + // overwrites an existing object; the full size is the delta. + #[cfg(not(feature = "enterprise"))] + let max_size = Some(ce_storage_quota_remaining(db, w_id, None).await? as usize); + #[cfg(feature = "enterprise")] + let max_size: Option = None; + + match upload_file_internal( + s3_client.clone(), + &file_key, + bytes_stream, + options, + max_size, + ) + .await + { + Ok((_, _size)) => { + #[cfg(not(feature = "enterprise"))] + bump_storage_usage( + db, + w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64, + ) + .await; + } + Err(e) => { + #[cfg(not(feature = "enterprise"))] + spawn_storage_usage_recount_floored(db, w_id); + return Err(e); + } + } files.entry(name).or_insert(vec![]).push(serde_json::json!({ "s3": &file_key diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 23f20459c3..c6d8e397f2 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -87,7 +87,8 @@ pub async fn upload_file_from_req( _file_key: &str, _req: axum::extract::Request, _options: PutMultipartOpts, -) -> error::Result { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) @@ -99,12 +100,77 @@ pub async fn upload_file_internal( _file_key: &str, _stream: impl Stream> + Unpin, _options: PutMultipartOpts, -) -> error::Result<()> { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } +// These stubs stand in for the CE quota helpers in a pure-OSS build; their only +// callers (apps.rs / args.rs uploads) are `not(enterprise)`, so gate them the +// same way — an enterprise-without-private build compiles neither. +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_storage_quota_remaining( + _db: &DB, + _w_id: &str, + _exclude_upload_id: Option<&str>, +) -> error::Result { + Ok(i64::MAX) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn reject_reserved_volume_key(_file_key: &str) -> error::Result<()> { + Ok(()) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub struct CeUploadBudget { + pub max_size: usize, + pub existing_size: i64, +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_upload_budget( + _db: &DB, + _w_id: &str, + _s3_client: &Arc, + _file_key: &str, + _content_length: Option, +) -> error::Result { + Ok(CeUploadBudget { max_size: usize::MAX, existing_size: 0 }) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn bump_storage_usage(_db: &DB, _w_id: &str, _storage: &str, _delta: i64) {} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn spawn_storage_usage_recount_floored(_db: &DB, _w_id: &str) {} + #[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn download_s3_file_internal( _authed: OptJobAuthed, diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index 329ea53e14..be49e3c564 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -1,7 +1,7 @@