From 89231e3f993a79261f961ab69a97777ade006195 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 31 Jul 2025 13:39:27 +0100 Subject: [PATCH] storcon: squash all migrations into one Problem Neon and Hadron deployments have the same database schema, but different migration histories. Some transactions have different identifiers too. If we don't do anything about it, then the storage controller would fail to apply the merged set of transactions. Summary of Changes We squash all migrations into a single one. If the schema already matches, then the new transaction gets applie without doing anything. For new regions, this migration will bootstrap the database schema. This should be merged in both neon and hadron codebases. Note that after deploying this change, the `__diesel_schema_migrations` table will still contain entries for the old pre-squash transactions. This is fine because diesel only considers the tranasactions embedded in the repo for application. Once we are certain that we are not going to roll back, we can clean up the `__diesel_schema_migrations` tables in prod, but this is manual and error prone, so I'd skip it. Rolling back Rolling back to a previous deployment which embeds the non-squashed transactions is safe. The old transactions are still present in `__diesel_schema_migrations` (i.e. considered applied), so no migrations will be run, so no migrations will be run. Note that this assumes that all transactions are applied and squashed into the new migration before deployment. --- .../down.sql | 1 - .../up.sql | 13 -- .../2024-01-07-212945_create_nodes/down.sql | 1 - .../2024-01-07-212945_create_nodes/up.sql | 10 -- .../down.sql | 2 - .../2024-02-29-094122_generations_null/up.sql | 4 - .../2024-03-18-184429_rename_policy/down.sql | 3 - .../2024-03-18-184429_rename_policy/up.sql | 3 - .../down.sql | 3 - .../2024-03-27-133204_tenant_policies/up.sql | 2 - .../down.sql | 1 - .../up.sql | 14 --- .../2024-07-26-140924_create_leader/down.sql | 1 - .../2024-07-26-140924_create_leader/up.sql | 5 - .../2024-08-23-102952_safekeepers/down.sql | 2 - .../2024-08-23-102952_safekeepers/up.sql | 15 --- .../down.sql | 2 - .../2024-08-23-170149_tenant_id_index/up.sql | 2 - .../2024-08-27-184400_pageserver_az/down.sql | 1 - .../2024-08-27-184400_pageserver_az/up.sql | 1 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 4 - .../up.sql | 1 - .../down.sql | 2 - .../up.sql | 2 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 2 - .../up.sql | 19 --- .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 1 - .../2025-03-18-103700_timeline_imports/up.sql | 6 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 1 - .../up.sql | 1 - .../down.sql | 2 - .../up.sql | 17 --- .../down.sql | 10 ++ .../up.sql | 112 ++++++++++++++++++ 50 files changed, 122 insertions(+), 160 deletions(-) delete mode 100644 storage_controller/migrations/2024-01-07-211257_create_tenant_shards/down.sql delete mode 100644 storage_controller/migrations/2024-01-07-211257_create_tenant_shards/up.sql delete mode 100644 storage_controller/migrations/2024-01-07-212945_create_nodes/down.sql delete mode 100644 storage_controller/migrations/2024-01-07-212945_create_nodes/up.sql delete mode 100644 storage_controller/migrations/2024-02-29-094122_generations_null/down.sql delete mode 100644 storage_controller/migrations/2024-02-29-094122_generations_null/up.sql delete mode 100644 storage_controller/migrations/2024-03-18-184429_rename_policy/down.sql delete mode 100644 storage_controller/migrations/2024-03-18-184429_rename_policy/up.sql delete mode 100644 storage_controller/migrations/2024-03-27-133204_tenant_policies/down.sql delete mode 100644 storage_controller/migrations/2024-03-27-133204_tenant_policies/up.sql delete mode 100644 storage_controller/migrations/2024-07-23-191537_create_metadata_health/down.sql delete mode 100644 storage_controller/migrations/2024-07-23-191537_create_metadata_health/up.sql delete mode 100644 storage_controller/migrations/2024-07-26-140924_create_leader/down.sql delete mode 100644 storage_controller/migrations/2024-07-26-140924_create_leader/up.sql delete mode 100644 storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql delete mode 100644 storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql delete mode 100644 storage_controller/migrations/2024-08-23-170149_tenant_id_index/down.sql delete mode 100644 storage_controller/migrations/2024-08-23-170149_tenant_id_index/up.sql delete mode 100644 storage_controller/migrations/2024-08-27-184400_pageserver_az/down.sql delete mode 100644 storage_controller/migrations/2024-08-27-184400_pageserver_az/up.sql delete mode 100644 storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql delete mode 100644 storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql delete mode 100644 storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql delete mode 100644 storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql delete mode 100644 storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql delete mode 100644 storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql delete mode 100644 storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql delete mode 100644 storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql delete mode 100644 storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/down.sql delete mode 100644 storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/up.sql delete mode 100644 storage_controller/migrations/2025-02-11-144848_pageserver_use_https/down.sql delete mode 100644 storage_controller/migrations/2025-02-11-144848_pageserver_use_https/up.sql delete mode 100644 storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/down.sql delete mode 100644 storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/up.sql delete mode 100644 storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/down.sql delete mode 100644 storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/up.sql delete mode 100644 storage_controller/migrations/2025-03-18-103700_timeline_imports/down.sql delete mode 100644 storage_controller/migrations/2025-03-18-103700_timeline_imports/up.sql delete mode 100644 storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/down.sql delete mode 100644 storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/up.sql delete mode 100644 storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/down.sql delete mode 100644 storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/up.sql delete mode 100644 storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/down.sql delete mode 100644 storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/up.sql delete mode 100644 storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/down.sql delete mode 100644 storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/up.sql delete mode 100644 storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/down.sql delete mode 100644 storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/up.sql create mode 100644 storage_controller/migrations/2025-07-31-115100_squash_migrations/down.sql create mode 100644 storage_controller/migrations/2025-07-31-115100_squash_migrations/up.sql diff --git a/storage_controller/migrations/2024-01-07-211257_create_tenant_shards/down.sql b/storage_controller/migrations/2024-01-07-211257_create_tenant_shards/down.sql deleted file mode 100644 index b875b91c00..0000000000 --- a/storage_controller/migrations/2024-01-07-211257_create_tenant_shards/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE tenant_shards; diff --git a/storage_controller/migrations/2024-01-07-211257_create_tenant_shards/up.sql b/storage_controller/migrations/2024-01-07-211257_create_tenant_shards/up.sql deleted file mode 100644 index 2ffdae6287..0000000000 --- a/storage_controller/migrations/2024-01-07-211257_create_tenant_shards/up.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE TABLE tenant_shards ( - tenant_id VARCHAR NOT NULL, - shard_number INTEGER NOT NULL, - shard_count INTEGER NOT NULL, - PRIMARY KEY(tenant_id, shard_number, shard_count), - shard_stripe_size INTEGER NOT NULL, - generation INTEGER NOT NULL, - generation_pageserver BIGINT NOT NULL, - placement_policy VARCHAR NOT NULL, - splitting SMALLINT NOT NULL, - -- config is JSON encoded, opaque to the database. - config TEXT NOT NULL -); \ No newline at end of file diff --git a/storage_controller/migrations/2024-01-07-212945_create_nodes/down.sql b/storage_controller/migrations/2024-01-07-212945_create_nodes/down.sql deleted file mode 100644 index ec303bc8cf..0000000000 --- a/storage_controller/migrations/2024-01-07-212945_create_nodes/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE nodes; diff --git a/storage_controller/migrations/2024-01-07-212945_create_nodes/up.sql b/storage_controller/migrations/2024-01-07-212945_create_nodes/up.sql deleted file mode 100644 index 9be0880fa4..0000000000 --- a/storage_controller/migrations/2024-01-07-212945_create_nodes/up.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TABLE nodes ( - node_id BIGINT PRIMARY KEY NOT NULL, - - scheduling_policy VARCHAR NOT NULL, - - listen_http_addr VARCHAR NOT NULL, - listen_http_port INTEGER NOT NULL, - listen_pg_addr VARCHAR NOT NULL, - listen_pg_port INTEGER NOT NULL -); \ No newline at end of file diff --git a/storage_controller/migrations/2024-02-29-094122_generations_null/down.sql b/storage_controller/migrations/2024-02-29-094122_generations_null/down.sql deleted file mode 100644 index 503231f69d..0000000000 --- a/storage_controller/migrations/2024-02-29-094122_generations_null/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE tenant_shards ALTER generation SET NOT NULL; -ALTER TABLE tenant_shards ALTER generation_pageserver SET NOT NULL; diff --git a/storage_controller/migrations/2024-02-29-094122_generations_null/up.sql b/storage_controller/migrations/2024-02-29-094122_generations_null/up.sql deleted file mode 100644 index 7e1e3cfe90..0000000000 --- a/storage_controller/migrations/2024-02-29-094122_generations_null/up.sql +++ /dev/null @@ -1,4 +0,0 @@ - - -ALTER TABLE tenant_shards ALTER generation DROP NOT NULL; -ALTER TABLE tenant_shards ALTER generation_pageserver DROP NOT NULL; \ No newline at end of file diff --git a/storage_controller/migrations/2024-03-18-184429_rename_policy/down.sql b/storage_controller/migrations/2024-03-18-184429_rename_policy/down.sql deleted file mode 100644 index 897c7e0d01..0000000000 --- a/storage_controller/migrations/2024-03-18-184429_rename_policy/down.sql +++ /dev/null @@ -1,3 +0,0 @@ - -UPDATE tenant_shards set placement_policy='{"Double": 1}' where placement_policy='{"Attached": 1}'; -UPDATE tenant_shards set placement_policy='"Single"' where placement_policy='{"Attached": 0}'; \ No newline at end of file diff --git a/storage_controller/migrations/2024-03-18-184429_rename_policy/up.sql b/storage_controller/migrations/2024-03-18-184429_rename_policy/up.sql deleted file mode 100644 index c898ac9aee..0000000000 --- a/storage_controller/migrations/2024-03-18-184429_rename_policy/up.sql +++ /dev/null @@ -1,3 +0,0 @@ - -UPDATE tenant_shards set placement_policy='{"Attached": 1}' where placement_policy='{"Double": 1}'; -UPDATE tenant_shards set placement_policy='{"Attached": 0}' where placement_policy='"Single"'; \ No newline at end of file diff --git a/storage_controller/migrations/2024-03-27-133204_tenant_policies/down.sql b/storage_controller/migrations/2024-03-27-133204_tenant_policies/down.sql deleted file mode 100644 index 33c06dc03d..0000000000 --- a/storage_controller/migrations/2024-03-27-133204_tenant_policies/down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This file should undo anything in `up.sql` - -ALTER TABLE tenant_shards drop scheduling_policy; \ No newline at end of file diff --git a/storage_controller/migrations/2024-03-27-133204_tenant_policies/up.sql b/storage_controller/migrations/2024-03-27-133204_tenant_policies/up.sql deleted file mode 100644 index aa00f0d2ca..0000000000 --- a/storage_controller/migrations/2024-03-27-133204_tenant_policies/up.sql +++ /dev/null @@ -1,2 +0,0 @@ - -ALTER TABLE tenant_shards add scheduling_policy VARCHAR NOT NULL DEFAULT '"Active"'; diff --git a/storage_controller/migrations/2024-07-23-191537_create_metadata_health/down.sql b/storage_controller/migrations/2024-07-23-191537_create_metadata_health/down.sql deleted file mode 100644 index 1ecfc8786f..0000000000 --- a/storage_controller/migrations/2024-07-23-191537_create_metadata_health/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE metadata_health; \ No newline at end of file diff --git a/storage_controller/migrations/2024-07-23-191537_create_metadata_health/up.sql b/storage_controller/migrations/2024-07-23-191537_create_metadata_health/up.sql deleted file mode 100644 index fa87eda119..0000000000 --- a/storage_controller/migrations/2024-07-23-191537_create_metadata_health/up.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE metadata_health ( - tenant_id VARCHAR NOT NULL, - shard_number INTEGER NOT NULL, - shard_count INTEGER NOT NULL, - PRIMARY KEY(tenant_id, shard_number, shard_count), - -- Rely on cascade behavior for delete - FOREIGN KEY(tenant_id, shard_number, shard_count) REFERENCES tenant_shards ON DELETE CASCADE, - healthy BOOLEAN NOT NULL DEFAULT TRUE, - last_scrubbed_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - - -INSERT INTO metadata_health(tenant_id, shard_number, shard_count) -SELECT tenant_id, shard_number, shard_count FROM tenant_shards; diff --git a/storage_controller/migrations/2024-07-26-140924_create_leader/down.sql b/storage_controller/migrations/2024-07-26-140924_create_leader/down.sql deleted file mode 100644 index 53222c614e..0000000000 --- a/storage_controller/migrations/2024-07-26-140924_create_leader/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE controllers; diff --git a/storage_controller/migrations/2024-07-26-140924_create_leader/up.sql b/storage_controller/migrations/2024-07-26-140924_create_leader/up.sql deleted file mode 100644 index 90546948cb..0000000000 --- a/storage_controller/migrations/2024-07-26-140924_create_leader/up.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE controllers ( - address VARCHAR NOT NULL, - started_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY(address, started_at) -); diff --git a/storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql b/storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql deleted file mode 100644 index 9dfc750586..0000000000 --- a/storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE safekeepers; diff --git a/storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql b/storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql deleted file mode 100644 index c78716660f..0000000000 --- a/storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql +++ /dev/null @@ -1,15 +0,0 @@ --- started out as a copy of cplane schema, removed the unnecessary columns. -CREATE TABLE safekeepers ( - -- the surrogate identifier defined by control plane database sequence - id BIGINT PRIMARY KEY, - region_id TEXT NOT NULL, - version BIGINT NOT NULL, - -- the natural id on whatever cloud platform, not needed in storage controller - -- instance_id TEXT UNIQUE NOT NULL, - host TEXT NOT NULL, - port INTEGER NOT NULL, - active BOOLEAN NOT NULL DEFAULT false, - -- projects_count INTEGER NOT NULL DEFAULT 0, - http_port INTEGER NOT NULL, - availability_zone_id TEXT NOT NULL -); diff --git a/storage_controller/migrations/2024-08-23-170149_tenant_id_index/down.sql b/storage_controller/migrations/2024-08-23-170149_tenant_id_index/down.sql deleted file mode 100644 index 518c747100..0000000000 --- a/storage_controller/migrations/2024-08-23-170149_tenant_id_index/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP INDEX tenant_shards_tenant_id; \ No newline at end of file diff --git a/storage_controller/migrations/2024-08-23-170149_tenant_id_index/up.sql b/storage_controller/migrations/2024-08-23-170149_tenant_id_index/up.sql deleted file mode 100644 index dd6b37781a..0000000000 --- a/storage_controller/migrations/2024-08-23-170149_tenant_id_index/up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Your SQL goes here -CREATE INDEX tenant_shards_tenant_id ON tenant_shards (tenant_id); \ No newline at end of file diff --git a/storage_controller/migrations/2024-08-27-184400_pageserver_az/down.sql b/storage_controller/migrations/2024-08-27-184400_pageserver_az/down.sql deleted file mode 100644 index 22df81c83c..0000000000 --- a/storage_controller/migrations/2024-08-27-184400_pageserver_az/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes DROP availability_zone_id; diff --git a/storage_controller/migrations/2024-08-27-184400_pageserver_az/up.sql b/storage_controller/migrations/2024-08-27-184400_pageserver_az/up.sql deleted file mode 100644 index 7112f92bf2..0000000000 --- a/storage_controller/migrations/2024-08-27-184400_pageserver_az/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes ADD availability_zone_id VARCHAR; diff --git a/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql b/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql deleted file mode 100644 index 4fcb928533..0000000000 --- a/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes ALTER availability_zone_id DROP NOT NULL; diff --git a/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql b/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql deleted file mode 100644 index c5b4534087..0000000000 --- a/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes ALTER availability_zone_id SET NOT NULL; diff --git a/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql b/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql deleted file mode 100644 index 127972a2e4..0000000000 --- a/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE tenant_shards DROP preferred_az_id; diff --git a/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql b/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql deleted file mode 100644 index 641a54feb2..0000000000 --- a/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE tenant_shards ADD preferred_az_id VARCHAR; diff --git a/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql b/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql deleted file mode 100644 index e26bff798f..0000000000 --- a/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE safekeepers DROP scheduling_policy; diff --git a/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql b/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql deleted file mode 100644 index d83cc6cc46..0000000000 --- a/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE safekeepers ADD scheduling_policy VARCHAR NOT NULL DEFAULT 'disabled'; diff --git a/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql b/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql deleted file mode 100644 index c2624f858b..0000000000 --- a/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql +++ /dev/null @@ -1,4 +0,0 @@ --- this sadly isn't a "true" revert of the migration, as the column is now at the end of the table. --- But preserving order is not a trivial operation. --- https://wiki.postgresql.org/wiki/Alter_column_position -ALTER TABLE safekeepers ADD active BOOLEAN NOT NULL DEFAULT false; diff --git a/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql b/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql deleted file mode 100644 index d76f044eda..0000000000 --- a/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE safekeepers DROP active; diff --git a/storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/down.sql b/storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/down.sql deleted file mode 100644 index 3c7126e343..0000000000 --- a/storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE safekeepers ALTER COLUMN scheduling_policy SET DEFAULT 'disabled'; -UPDATE safekeepers SET scheduling_policy = 'disabled' WHERE scheduling_policy = 'pause'; diff --git a/storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/up.sql b/storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/up.sql deleted file mode 100644 index 9ff75444f3..0000000000 --- a/storage_controller/migrations/2025-01-15-181207_safekeepers_disabled_to_pause/up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE safekeepers ALTER COLUMN scheduling_policy SET DEFAULT 'pause'; -UPDATE safekeepers SET scheduling_policy = 'pause' WHERE scheduling_policy = 'disabled'; diff --git a/storage_controller/migrations/2025-02-11-144848_pageserver_use_https/down.sql b/storage_controller/migrations/2025-02-11-144848_pageserver_use_https/down.sql deleted file mode 100644 index 0f051d3ac3..0000000000 --- a/storage_controller/migrations/2025-02-11-144848_pageserver_use_https/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes DROP listen_https_port; diff --git a/storage_controller/migrations/2025-02-11-144848_pageserver_use_https/up.sql b/storage_controller/migrations/2025-02-11-144848_pageserver_use_https/up.sql deleted file mode 100644 index 172237d477..0000000000 --- a/storage_controller/migrations/2025-02-11-144848_pageserver_use_https/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes ADD listen_https_port INTEGER; diff --git a/storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/down.sql b/storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/down.sql deleted file mode 100644 index 8f75e8947e..0000000000 --- a/storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE timelines; -DROP TABLE safekeeper_timeline_pending_ops; diff --git a/storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/up.sql b/storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/up.sql deleted file mode 100644 index 82003ab292..0000000000 --- a/storage_controller/migrations/2025-02-14-160526_safekeeper_timelines/up.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TABLE timelines ( - tenant_id VARCHAR NOT NULL, - timeline_id VARCHAR NOT NULL, - start_lsn pg_lsn NOT NULL, - generation INTEGER NOT NULL, - sk_set BIGINT[] NOT NULL, - new_sk_set BIGINT[], - cplane_notified_generation INTEGER NOT NULL, - deleted_at timestamptz, - PRIMARY KEY(tenant_id, timeline_id) -); -CREATE TABLE safekeeper_timeline_pending_ops ( - sk_id BIGINT NOT NULL, - tenant_id VARCHAR NOT NULL, - timeline_id VARCHAR NOT NULL, - generation INTEGER NOT NULL, - op_kind VARCHAR NOT NULL, - PRIMARY KEY(tenant_id, timeline_id, sk_id) -); diff --git a/storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/down.sql b/storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/down.sql deleted file mode 100644 index 378e9f8c16..0000000000 --- a/storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE safekeepers DROP https_port; diff --git a/storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/up.sql b/storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/up.sql deleted file mode 100644 index bb47b0b256..0000000000 --- a/storage_controller/migrations/2025-02-28-141741_safekeeper_use_https/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE safekeepers ADD https_port INTEGER; diff --git a/storage_controller/migrations/2025-03-18-103700_timeline_imports/down.sql b/storage_controller/migrations/2025-03-18-103700_timeline_imports/down.sql deleted file mode 100644 index 4e7ae74ce2..0000000000 --- a/storage_controller/migrations/2025-03-18-103700_timeline_imports/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE timeline_imports; diff --git a/storage_controller/migrations/2025-03-18-103700_timeline_imports/up.sql b/storage_controller/migrations/2025-03-18-103700_timeline_imports/up.sql deleted file mode 100644 index 27741c439d..0000000000 --- a/storage_controller/migrations/2025-03-18-103700_timeline_imports/up.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE timeline_imports ( - tenant_id VARCHAR NOT NULL, - timeline_id VARCHAR NOT NULL, - shard_statuses JSONB NOT NULL, - PRIMARY KEY(tenant_id, timeline_id) -); diff --git a/storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/down.sql b/storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/down.sql deleted file mode 100644 index a09acb916b..0000000000 --- a/storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes DROP COLUMN lifecycle; diff --git a/storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/up.sql b/storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/up.sql deleted file mode 100644 index e03a0cadba..0000000000 --- a/storage_controller/migrations/2025-06-01-201442_add_lifecycle_to_nodes/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes ADD COLUMN lifecycle VARCHAR NOT NULL DEFAULT 'active'; diff --git a/storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/down.sql b/storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/down.sql deleted file mode 100644 index f9f2ebb070..0000000000 --- a/storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes DROP listen_grpc_addr, listen_grpc_port; diff --git a/storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/up.sql b/storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/up.sql deleted file mode 100644 index 8291864b16..0000000000 --- a/storage_controller/migrations/2025-06-17-082247_pageserver_grpc_addr/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nodes ADD listen_grpc_addr VARCHAR NULL, ADD listen_grpc_port INTEGER NULL; diff --git a/storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/down.sql b/storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/down.sql deleted file mode 100644 index bc9b501189..0000000000 --- a/storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE safekeepers ALTER COLUMN scheduling_policy SET DEFAULT 'pause'; \ No newline at end of file diff --git a/storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/up.sql b/storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/up.sql deleted file mode 100644 index 18c89bed7b..0000000000 --- a/storage_controller/migrations/2025-07-02-170751_safekeeper_default_no_pause/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE safekeepers ALTER COLUMN scheduling_policy SET DEFAULT 'activating'; \ No newline at end of file diff --git a/storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/down.sql b/storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/down.sql deleted file mode 100644 index 27d6048cd3..0000000000 --- a/storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE timelines DROP sk_set_notified_generation; diff --git a/storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/up.sql b/storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/up.sql deleted file mode 100644 index 50178ab6a3..0000000000 --- a/storage_controller/migrations/2025-07-08-114340_sk_set_notified_generation/up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE timelines ADD sk_set_notified_generation INTEGER NOT NULL DEFAULT 1; diff --git a/storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/down.sql b/storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/down.sql deleted file mode 100644 index b45b45e438..0000000000 --- a/storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE hadron_safekeepers; -DROP TABLE hadron_timeline_safekeepers; diff --git a/storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/up.sql b/storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/up.sql deleted file mode 100644 index 6cee981efc..0000000000 --- a/storage_controller/migrations/2025-07-17-000001_hadron_safekeepers/up.sql +++ /dev/null @@ -1,17 +0,0 @@ --- hadron_safekeepers keep track of all Safe Keeper nodes that exist in the system. --- Upon startup, each Safe Keeper reaches out to the hadron cluster coordinator to register its node ID and listen addresses. - -CREATE TABLE hadron_safekeepers ( - sk_node_id BIGINT PRIMARY KEY NOT NULL, - listen_http_addr VARCHAR NOT NULL, - listen_http_port INTEGER NOT NULL, - listen_pg_addr VARCHAR NOT NULL, - listen_pg_port INTEGER NOT NULL -); - -CREATE TABLE hadron_timeline_safekeepers ( - timeline_id VARCHAR NOT NULL, - sk_node_id BIGINT NOT NULL, - legacy_endpoint_id UUID DEFAULT NULL, - PRIMARY KEY(timeline_id, sk_node_id) -); diff --git a/storage_controller/migrations/2025-07-31-115100_squash_migrations/down.sql b/storage_controller/migrations/2025-07-31-115100_squash_migrations/down.sql new file mode 100644 index 0000000000..b123ba532c --- /dev/null +++ b/storage_controller/migrations/2025-07-31-115100_squash_migrations/down.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS timelines CASCADE; +DROP TABLE IF EXISTS timeline_imports CASCADE; +DROP TABLE IF EXISTS tenant_shards CASCADE; +DROP TABLE IF EXISTS safekeepers CASCADE; +DROP TABLE IF EXISTS safekeeper_timeline_pending_ops CASCADE; +DROP TABLE IF EXISTS nodes CASCADE; +DROP TABLE IF EXISTS metadata_health CASCADE; +DROP TABLE IF EXISTS hadron_timeline_safekeepers CASCADE; +DROP TABLE IF EXISTS hadron_safekeepers CASCADE; +DROP TABLE IF EXISTS controllers CASCADE; diff --git a/storage_controller/migrations/2025-07-31-115100_squash_migrations/up.sql b/storage_controller/migrations/2025-07-31-115100_squash_migrations/up.sql new file mode 100644 index 0000000000..c662b40a8e --- /dev/null +++ b/storage_controller/migrations/2025-07-31-115100_squash_migrations/up.sql @@ -0,0 +1,112 @@ +/* + * This migration squashes all previous migrations into a single one. + * It will be applied in two cases: + * 1. Spinning up a new region. There's no previous migrations apart from the diesel setup in this case + * 2. For existing regions. + * + * Hence, this migration does nothing if the schema is already up to date. + */ + +CREATE TABLE IF NOT EXISTS controllers ( + address character varying NOT NULL, + started_at timestamp with time zone NOT NULL, + PRIMARY KEY(address, started_at) +); + +CREATE TABLE IF NOT EXISTS hadron_safekeepers ( + sk_node_id bigint PRIMARY KEY NOT NULL, + listen_http_addr character varying NOT NULL, + listen_http_port integer NOT NULL, + listen_pg_addr character varying NOT NULL, + listen_pg_port integer NOT NULL +); + +CREATE TABLE IF NOT EXISTS hadron_timeline_safekeepers ( + timeline_id character varying NOT NULL, + sk_node_id bigint NOT NULL, + legacy_endpoint_id uuid, + PRIMARY KEY(timeline_id, sk_node_id) +); + +CREATE TABLE IF NOT EXISTS tenant_shards ( + tenant_id character varying NOT NULL, + shard_number integer NOT NULL, + shard_count integer NOT NULL, + PRIMARY KEY(tenant_id, shard_number, shard_count), + shard_stripe_size integer NOT NULL, + generation integer, + generation_pageserver bigint, + placement_policy character varying NOT NULL, + splitting smallint NOT NULL, + config text NOT NULL, + scheduling_policy character varying DEFAULT '"Active"'::character varying NOT NULL, + preferred_az_id character varying +); + +CREATE INDEX IF NOT EXISTS tenant_shards_tenant_id ON tenant_shards USING btree (tenant_id); + +CREATE TABLE IF NOT EXISTS metadata_health ( + tenant_id character varying NOT NULL, + shard_number integer NOT NULL, + shard_count integer NOT NULL, + PRIMARY KEY(tenant_id, shard_number, shard_count), + -- Rely on cascade behavior for delete + FOREIGN KEY(tenant_id, shard_number, shard_count) REFERENCES tenant_shards ON DELETE CASCADE, + healthy boolean DEFAULT true NOT NULL, + last_scrubbed_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE IF NOT EXISTS nodes ( + node_id bigint PRIMARY KEY NOT NULL, + scheduling_policy character varying NOT NULL, + listen_http_addr character varying NOT NULL, + listen_http_port integer NOT NULL, + listen_pg_addr character varying NOT NULL, + listen_pg_port integer NOT NULL, + availability_zone_id character varying NOT NULL, + listen_https_port integer, + lifecycle character varying DEFAULT 'active'::character varying NOT NULL, + listen_grpc_addr character varying, + listen_grpc_port integer +); + +CREATE TABLE IF NOT EXISTS safekeeper_timeline_pending_ops ( + sk_id bigint NOT NULL, + tenant_id character varying NOT NULL, + timeline_id character varying NOT NULL, + generation integer NOT NULL, + op_kind character varying NOT NULL, + PRIMARY KEY(tenant_id, timeline_id, sk_id) +); + +CREATE TABLE IF NOT EXISTS safekeepers ( + id bigint PRIMARY KEY NOT NULL, + region_id text NOT NULL, + version bigint NOT NULL, + host text NOT NULL, + port integer NOT NULL, + http_port integer NOT NULL, + availability_zone_id text NOT NULL, + scheduling_policy character varying DEFAULT 'activating'::character varying NOT NULL, + https_port integer +); + +CREATE TABLE IF NOT EXISTS timeline_imports ( + tenant_id character varying NOT NULL, + timeline_id character varying NOT NULL, + shard_statuses jsonb NOT NULL, + PRIMARY KEY(tenant_id, timeline_id) +); + +CREATE TABLE IF NOT EXISTS timelines ( + tenant_id character varying NOT NULL, + timeline_id character varying NOT NULL, + start_lsn pg_lsn NOT NULL, + generation integer NOT NULL, + sk_set bigint[] NOT NULL, + new_sk_set bigint[], + cplane_notified_generation integer NOT NULL, + deleted_at timestamp with time zone, + sk_set_notified_generation integer DEFAULT 1 NOT NULL, + PRIMARY KEY(tenant_id, timeline_id) +);