From f818817e10a99e2beac6b2529acbeaf730dc3e6b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 5 Feb 2024 08:55:39 +0100 Subject: [PATCH] feat: add nobypassrls migration --- ...dea8cafd0043634f61b0f652a93767c9c30a.json} | 12 +- ...ddd7cd50a4782101e971175d0c5c798593a40.json | 23 ++++ ...5850e12d2f717d7b8e430fa2508d2bfff9a58.json | 12 ++ ...17e87df7f8dfc5b03c1c77f5bebe6e4df7cf2.json | 20 +++ ...3c17feccb538ea39eca9db7d8f36dfd61aec6.json | 20 --- backend/custom_migrations/bypassrls_1.sql | 14 +++ .../20240204212055_auto_bypassrls.down.sql | 1 + .../20240204212055_auto_bypassrls.up.sql | 2 + backend/src/main.rs | 29 ----- backend/windmill-api/src/db.rs | 114 +++++++++++++++++- 10 files changed, 193 insertions(+), 54 deletions(-) rename backend/.sqlx/{query-917afc04613a7d4373c9152b8ecaa946a2abfb3e5be437ee3b0aa05b3bd39702.json => query-18699cb0eca25b6bde05d81571dfdea8cafd0043634f61b0f652a93767c9c30a.json} (72%) create mode 100644 backend/.sqlx/query-611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40.json create mode 100644 backend/.sqlx/query-b728231a89726207e3b135b98315850e12d2f717d7b8e430fa2508d2bfff9a58.json create mode 100644 backend/.sqlx/query-bcdde5a590a056034c811c2109a17e87df7f8dfc5b03c1c77f5bebe6e4df7cf2.json delete mode 100644 backend/.sqlx/query-f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6.json create mode 100644 backend/custom_migrations/bypassrls_1.sql create mode 100644 backend/migrations/20240204212055_auto_bypassrls.down.sql create mode 100644 backend/migrations/20240204212055_auto_bypassrls.up.sql diff --git a/backend/.sqlx/query-917afc04613a7d4373c9152b8ecaa946a2abfb3e5be437ee3b0aa05b3bd39702.json b/backend/.sqlx/query-18699cb0eca25b6bde05d81571dfdea8cafd0043634f61b0f652a93767c9c30a.json similarity index 72% rename from backend/.sqlx/query-917afc04613a7d4373c9152b8ecaa946a2abfb3e5be437ee3b0aa05b3bd39702.json rename to backend/.sqlx/query-18699cb0eca25b6bde05d81571dfdea8cafd0043634f61b0f652a93767c9c30a.json index b7c541f4b0..6ad0bd2efd 100644 --- a/backend/.sqlx/query-917afc04613a7d4373c9152b8ecaa946a2abfb3e5be437ee3b0aa05b3bd39702.json +++ b/backend/.sqlx/query-18699cb0eca25b6bde05d81571dfdea8cafd0043634f61b0f652a93767c9c30a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE queue SET canceled = true, canceled_by = $2, scheduled_for = now(), suspend = 0 WHERE scheduled_for < now() AND workspace_id = $1 AND schedule_path IS NULL RETURNING id, running", + "query": "UPDATE queue SET canceled = true, canceled_by = $2, scheduled_for = now(), suspend = 0 WHERE scheduled_for < now() AND workspace_id = $1 AND schedule_path IS NULL RETURNING id, running, is_flow_step", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "running", "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "is_flow_step", + "type_info": "Bool" } ], "parameters": { @@ -22,8 +27,9 @@ }, "nullable": [ false, - false + false, + true ] }, - "hash": "917afc04613a7d4373c9152b8ecaa946a2abfb3e5be437ee3b0aa05b3bd39702" + "hash": "18699cb0eca25b6bde05d81571dfdea8cafd0043634f61b0f652a93767c9c30a" } diff --git a/backend/.sqlx/query-611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40.json b/backend/.sqlx/query-611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40.json new file mode 100644 index 0000000000..679b05d29d --- /dev/null +++ b/backend/.sqlx/query-611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM queue WHERE workspace_id = $1 and root_job = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "611e3cd49d38a37db8912397c4eddd7cd50a4782101e971175d0c5c798593a40" +} diff --git a/backend/.sqlx/query-b728231a89726207e3b135b98315850e12d2f717d7b8e430fa2508d2bfff9a58.json b/backend/.sqlx/query-b728231a89726207e3b135b98315850e12d2f717d7b8e430fa2508d2bfff9a58.json new file mode 100644 index 0000000000..dd91b59c59 --- /dev/null +++ b/backend/.sqlx/query-b728231a89726207e3b135b98315850e12d2f717d7b8e430fa2508d2bfff9a58.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO windmill_migrations (name) VALUES ('bypassrls_1')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "b728231a89726207e3b135b98315850e12d2f717d7b8e430fa2508d2bfff9a58" +} diff --git a/backend/.sqlx/query-bcdde5a590a056034c811c2109a17e87df7f8dfc5b03c1c77f5bebe6e4df7cf2.json b/backend/.sqlx/query-bcdde5a590a056034c811c2109a17e87df7f8dfc5b03c1c77f5bebe6e4df7cf2.json new file mode 100644 index 0000000000..b9ffb1b182 --- /dev/null +++ b/backend/.sqlx/query-bcdde5a590a056034c811c2109a17e87df7f8dfc5b03c1c77f5bebe6e4df7cf2.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'bypassrls_1')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "bcdde5a590a056034c811c2109a17e87df7f8dfc5b03c1c77f5bebe6e4df7cf2" +} diff --git a/backend/.sqlx/query-f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6.json b/backend/.sqlx/query-f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6.json deleted file mode 100644 index 74715d1955..0000000000 --- a/backend/.sqlx/query-f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select version from _sqlx_migrations order by version desc limit 1;", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "version", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "f8696ade96d00b7e6660c44d7443c17feccb538ea39eca9db7d8f36dfd61aec6" -} diff --git a/backend/custom_migrations/bypassrls_1.sql b/backend/custom_migrations/bypassrls_1.sql new file mode 100644 index 0000000000..1918b26b0b --- /dev/null +++ b/backend/custom_migrations/bypassrls_1.sql @@ -0,0 +1,14 @@ +CREATE POLICY admin_policy ON account TO windmill_admin USING (true); +CREATE POLICY admin_policy ON app TO windmill_admin USING (true); +CREATE POLICY admin_policy ON audit TO windmill_admin USING (true); +CREATE POLICY admin_policy ON capture TO windmill_admin USING (true); +CREATE POLICY admin_policy ON completed_job TO windmill_admin USING (true); +CREATE POLICY admin_policy ON flow TO windmill_admin USING (true); +CREATE POLICY admin_policy ON folder TO windmill_admin USING (true); +CREATE POLICY admin_policy ON queue TO windmill_admin USING (true); +CREATE POLICY admin_policy ON raw_app TO windmill_admin USING (true); +CREATE POLICY admin_policy ON resource TO windmill_admin USING (true); +CREATE POLICY admin_policy ON schedule TO windmill_admin USING (true); +CREATE POLICY admin_policy ON script TO windmill_admin USING (true); +CREATE POLICY admin_policy ON usr_to_group TO windmill_admin USING (true); +CREATE POLICY admin_policy ON variable TO windmill_admin USING (true); \ No newline at end of file diff --git a/backend/migrations/20240204212055_auto_bypassrls.down.sql b/backend/migrations/20240204212055_auto_bypassrls.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20240204212055_auto_bypassrls.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20240204212055_auto_bypassrls.up.sql b/backend/migrations/20240204212055_auto_bypassrls.up.sql new file mode 100644 index 0000000000..a215298cc4 --- /dev/null +++ b/backend/migrations/20240204212055_auto_bypassrls.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +create table windmill_migrations (name text primary key, created_at timestamp default current_timestamp); \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 1aba22c6d5..cca5e595ff 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -211,37 +211,8 @@ async fn main() -> anyhow::Result<()> { let is_agent = mode == Mode::Agent; if !is_agent { - let last_mig_version = sqlx::query_scalar!( - "select version from _sqlx_migrations order by version desc limit 1;" - ) - .fetch_optional(&db) - .await - .ok() - .flatten(); - - tracing::info!( - "Last migration version: {last_mig_version:?}. Starting potential migration of the db if first connection on a new windmill version (can take a while depending on the migration) ...", - ); - // migration code to avoid break windmill_api::migrate_db(&db).await?; - - let new_last_mig_version = sqlx::query_scalar!( - "select version from _sqlx_migrations order by version desc limit 1;" - ) - .fetch_optional(&db) - .await - .ok() - .flatten(); - - if last_mig_version != new_last_mig_version { - tracing::info!( - "Completed migration of the db. New migration version: {}", - new_last_mig_version.unwrap_or(-1) - ); - } else { - tracing::info!("No migration, db was up-to-date"); - } } let (killpill_tx, killpill_rx) = tokio::sync::broadcast::channel::<()>(2); let (killpill_phase2_tx, killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2); diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index cc1b652ee6..ce60abc94a 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use sqlx::{Pool, Postgres}; +use sqlx::{migrate::Migrate, pool::PoolConnection, Executor, Pool, Postgres}; use windmill_common::{ db::{Authable, Authed}, error::Error, @@ -14,8 +14,85 @@ use windmill_common::{ pub type DB = Pool; +struct CustomMigrator { + inner: PoolConnection, +} +impl Migrate for CustomMigrator { + fn ensure_migrations_table( + &mut self, + ) -> futures::prelude::future::BoxFuture<'_, Result<(), sqlx::migrate::MigrateError>> { + self.inner.ensure_migrations_table() + } + + fn dirty_version( + &mut self, + ) -> futures::prelude::future::BoxFuture<'_, Result, sqlx::migrate::MigrateError>> + { + self.inner.dirty_version() + } + + fn list_applied_migrations( + &mut self, + ) -> futures::prelude::future::BoxFuture< + '_, + Result, sqlx::migrate::MigrateError>, + > { + self.inner.list_applied_migrations() + } + + fn lock( + &mut self, + ) -> futures::prelude::future::BoxFuture<'_, Result<(), sqlx::migrate::MigrateError>> { + tracing::info!("Started locking PG for migration purposes"); + let r = self.inner.lock(); + tracing::info!("Locked PG for migration purposes"); + r + } + + fn unlock( + &mut self, + ) -> futures::prelude::future::BoxFuture<'_, Result<(), sqlx::migrate::MigrateError>> { + tracing::info!("Started unlocking PG for migration purposes"); + let r = self.inner.unlock(); + tracing::info!("Unlocked PG for migration purposes"); + r + } + + fn apply<'e: 'm, 'm>( + &'e mut self, + migration: &'m sqlx::migrate::Migration, + ) -> futures::prelude::future::BoxFuture< + 'm, + Result, + > { + tracing::info!( + "Started applying migration {}: {}", + migration.version, + migration.description + ); + let r = self.inner.apply(migration); + tracing::info!("Finished applying migration {}", migration.version); + r + } + + fn revert<'e: 'm, 'm>( + &'e mut self, + migration: &'m sqlx::migrate::Migration, + ) -> futures::prelude::future::BoxFuture< + 'm, + Result, + > { + self.inner.revert(migration) + } +} + pub async fn migrate(db: &DB) -> Result<(), Error> { - match sqlx::migrate!("../migrations").run(db).await { + let migrator = db.acquire().await?; + let mut custom_migrator = CustomMigrator { inner: migrator }; + match sqlx::migrate!("../migrations") + .run_direct(&mut custom_migrator) + .await + { Ok(_) => Ok(()), Err(sqlx::migrate::MigrateError::VersionMissing(e)) => { tracing::error!("Database had been applied more migrations than this container. @@ -26,6 +103,39 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { Err(err) => Err(err), }?; + if let Err(e) = windmill_migrations(&mut custom_migrator, db).await { + tracing::error!("Could not apply windmill custom migrations: {e}") + } + + Ok(()) +} + +async fn windmill_migrations(migrator: &mut CustomMigrator, db: &DB) -> Result<(), Error> { + if std::env::var("MIGRATION_NO_BYPASSRLS").is_ok() { + #[cfg(feature = "enterprise")] + { + migrator.lock().await?; + let mut tx = db.begin().await?; + let has_done_migration = sqlx::query_scalar!( + "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'bypassrls_1')", + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if !has_done_migration { + let query = include_str!("../../custom_migrations/bypassrls_1.sql"); + tracing::info!("Applying bypassrls_1.sql"); + tx.execute_many(query); + tracing::info!("Applied bypassrls_1.sql"); + sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('bypassrls_1')") + .execute(&mut *tx) + .await?; + tx.commit().await?; + } + migrator.unlock().await?; + } + } Ok(()) }