diff --git a/backend/.sqlx/query-1ddb83c0941de9ca80f867b57ad20c3ee0952d118d339f94d16102d9edb23ec0.json b/backend/.sqlx/query-1ddb83c0941de9ca80f867b57ad20c3ee0952d118d339f94d16102d9edb23ec0.json new file mode 100644 index 0000000000..123c9fc89b --- /dev/null +++ b/backend/.sqlx/query-1ddb83c0941de9ca80f867b57ad20c3ee0952d118d339f94d16102d9edb23ec0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT resource_type FROM resource WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "resource_type", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1ddb83c0941de9ca80f867b57ad20c3ee0952d118d339f94d16102d9edb23ec0" +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 41702acba7..7b2f5626fd 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -31949,6 +31949,9 @@ components: description: the resolved resource, rendered into profiles.yml target: type: string + resource_type: + type: string + description: decides whether the value is translated into a profiles.yml target or already is one required: - value diff --git a/backend/windmill-api/src/dbt.rs b/backend/windmill-api/src/dbt.rs index 8891834c2f..ccdd189f66 100644 --- a/backend/windmill-api/src/dbt.rs +++ b/backend/windmill-api/src/dbt.rs @@ -60,7 +60,8 @@ async fn get_warehouse( "the dbt warehouse `{name}` points at `{resource_path}`, which does not exist" )) })?; - return Ok(Json(DbtWarehouseConnection { value, target })); + let resource_type = warehouse_resource_type(&db, &w_id, &resource_path).await?; + return Ok(Json(DbtWarehouseConnection { value, target, resource_type })); } return Err(Error::BadRequest( "this route resolves a dbt warehouse for a running job and needs a job token" @@ -102,7 +103,22 @@ async fn get_warehouse( "the dbt warehouse `{name}` points at `{resource_path}`, which does not exist" )) })?; - Ok(Json(DbtWarehouseConnection { value, target })) + let resource_type = warehouse_resource_type(&db, &w_id, &resource_path).await?; + Ok(Json(DbtWarehouseConnection { value, target, resource_type })) +} + +/// A warehouse resource's type, which decides whether its value is translated +/// into a `profiles.yml` target or taken as one. Read separately from the value +/// because the interpolating loader returns the value alone. +async fn warehouse_resource_type(db: &DB, w_id: &str, path: &str) -> Result { + sqlx::query_scalar!( + "SELECT resource_type FROM resource WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("the dbt warehouse points at `{path}`, which does not exist"))) } /// A settled node's state, for a worker that cannot write the database. diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index e55dfe3b57..ddf2af3a26 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -2546,8 +2546,18 @@ pub struct DbtWarehouseConnection { /// schema generated clients validate against. #[serde(default, skip_serializing_if = "Option::is_none")] pub target: Option, + /// What the value IS, which its shape cannot say: a `dbt_profile`'s value is a + /// `profiles.yml` output block and every other type's is a connection to translate, + /// and both are objects carrying a `type`. Defaulted so a worker still resolves + /// against a server predating the field — which serves no `dbt_profile` anyway. + #[serde(default)] + pub resource_type: String, } +/// The resource type whose value is a `profiles.yml` output block, taken as it +/// is rather than translated. +pub const DBT_PROFILE_RESOURCE_TYPE: &str = "dbt_profile"; + /// The warehouse a dbt project runs against, by name — `main` when the /// descriptor names none. /// diff --git a/backend/windmill-worker/src/dbt_engine.rs b/backend/windmill-worker/src/dbt_engine.rs index 6ee8325ddf..2da3481b12 100644 --- a/backend/windmill-worker/src/dbt_engine.rs +++ b/backend/windmill-worker/src/dbt_engine.rs @@ -28,6 +28,14 @@ use crate::handle_child::{get_mem_peak, run_future_with_polling_update_job_polle lazy_static::lazy_static! { pub static ref DBT_CACHE_DIR: String = format!("{}dbt", *ROOT_CACHE_NOMOUNT_DIR); + /// Adapters an operator vouches for beyond `PUBLISHED_ADAPTERS`, comma-separated — so a + /// brand-new adapter needs an admin's decision, not a Windmill release. + static ref DBT_EXTRA_ADAPTERS: Vec = std::env::var("DBT_EXTRA_ADAPTERS") + .unwrap_or_default() + .split(',') + .map(|a| a.trim().to_ascii_lowercase()) + .filter(|a| !a.is_empty()) + .collect(); /// Where an operator may pre-stage an Apache-2.0 engine in a derived image. /// A persistent image path, unlike the runtime caches, which are a fresh /// volume at start — which is the whole reason it is a separate directory. @@ -141,6 +149,36 @@ fn checked_version<'a>(v: Option<&'a str>, field: &str) -> error::Result` comes from an adapter name a +/// SCRIPT AUTHOR chooses, `dbt-` is not a reserved PyPI prefix, and this install runs outside +/// the nsjail ordinary dependency installation uses — so an unbounded name would run a PEP +/// 517 backend as the worker. The admin decides what is trusted, via `DBT_EXTRA_ADAPTERS`. +fn ensure_adapter_installable(adapter: &DbtAdapter) -> error::Result<()> { + let name = adapter.name(); + if adapter.known().is_some() + || PUBLISHED_ADAPTERS.contains(&name) + || DBT_EXTRA_ADAPTERS.iter().any(|a| a == name) + { + return Ok(()); + } + Err(Error::BadRequest(format!( + "`{name}` is not an adapter this instance installs: the dbt-core 1.x engine would have \ + to fetch `dbt-{name}` from PyPI, and `dbt-` is not a reserved name there. An admin adds \ + it to DBT_EXTRA_ADAPTERS, or use an engine that ships its adapters (`engine: fusion`)" + ))) +} + /// A uv venv per (dbt version, adapter): the adapter is a separate pip package /// and installing every adapter into one venv would make their transitive /// dependency sets fight. @@ -153,6 +191,7 @@ async fn provision_core_1x( conn: &Connection, ctx: &mut JobCtx<'_>, ) -> error::Result { + ensure_adapter_installable(&adapter)?; if adapter.pip_package().is_empty() { return Err(Error::BadRequest(format!( "the {} adapter has no dbt-core 1.x package: it exists only inside the Fusion \ @@ -722,6 +761,7 @@ async fn run_tool( #[cfg(test)] mod core1x_tests { use super::*; + use crate::dbt_profiles::KnownAdapter; // Several adapters cap dbt-core below what this runtime would ask for // (dbt-mysql ~=1.7, dbt-oracle and dbt-databricks below 1.12) and @@ -729,8 +769,8 @@ mod core1x_tests { // those projects fail at provisioning. The install names a ceiling instead. #[test] fn every_adapter_either_names_a_package_or_is_fusion_only() { - for a in DbtAdapter::ALL { - if matches!(a, DbtAdapter::Salesforce) { + for a in KnownAdapter::ALL { + if matches!(a, KnownAdapter::Salesforce) { continue; } assert!( @@ -740,8 +780,21 @@ mod core1x_tests { ); } // Fusion has it built in, and there is no package to install. - assert!(DbtAdapter::Salesforce.pip_package().is_empty()); - assert_eq!(DbtAdapter::Salesforce.name(), "salesforce"); + assert!(KnownAdapter::Salesforce.pip_package().is_empty()); + assert_eq!(KnownAdapter::Salesforce.name(), "salesforce"); + } + + // `dbt-` is not a reserved prefix on PyPI and this install is not sandboxed, + // so the name a script author picks decides which package runs its build + // backend as the worker. + #[test] + fn an_unvouched_adapter_is_not_installed() { + let known = DbtAdapter::from_dbt_type("postgres").unwrap(); + assert!(ensure_adapter_installable(&known).is_ok()); + let published = DbtAdapter::from_dbt_type("trino").unwrap(); + assert!(ensure_adapter_installable(&published).is_ok()); + let squatted = DbtAdapter::from_dbt_type("totally-legit-adapter").unwrap(); + assert!(ensure_adapter_installable(&squatted).is_err()); } /// A preview submits its own lockfile, so this string reaches a path join diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index e3e9ee6817..33ddd6c32e 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -34,7 +34,9 @@ use crate::common::{ }; use crate::common::{start_child_process, OccupancyMetrics}; use crate::dbt_engine::{provision_engine, ProvisionedEngine, DBT_CACHE_DIR}; -use crate::dbt_profiles::{ensure_adapter_licensed, render_profile, DbtAdapter}; +use crate::dbt_profiles::{ + ensure_adapter_licensed, render_dbt_profile, render_profile, DbtAdapter, KnownAdapter, +}; use crate::handle_child::{ get_mem_peak, handle_child, run_future_with_polling_update_job_poller, JobCtx, JobDeadline, }; @@ -1541,15 +1543,7 @@ async fn write_profiles( .profile .adapter .as_deref() - .map(|t| { - DbtAdapter::from_resource_type(t).ok_or_else(|| { - Error::BadRequest(format!( - "`profile.type: {t}` is not a supported dbt adapter (postgres, redshift, \ - mysql, duckdb, clickhouse, snowflake, bigquery, databricks, salesforce, \ - mssql, oracle)" - )) - }) - }) + .map(DbtAdapter::from_dbt_type) .transpose()?; if let Some(own) = descriptor.profile.profiles_yml.as_deref() { @@ -1573,7 +1567,7 @@ async fn write_profiles( ) .await?; let actual = target.adapter; - if let Some(declared) = declared.filter(|d| *d != actual) { + if let Some(declared) = declared.filter(|d| *d != actual).as_ref() { return Err(Error::BadRequest(format!( "`profile.type: {}` disagrees with `{}`, whose target uses `{}`. dbt connects \ with the file, so remove `profile.type` or correct it", @@ -1583,7 +1577,7 @@ async fn write_profiles( ))); } let adapter = actual; - ensure_adapter_licensed(adapter)?; + ensure_adapter_licensed(&adapter)?; // The target's own database and schema, read from the file dbt connects // with. A relation that sits in them is then spelled plainly, exactly as // a workspace-warehouse project spells it, and one that overrides them @@ -1625,18 +1619,43 @@ async fn write_profiles( )); } + use windmill_common::workspaces::DBT_PROFILE_RESOURCE_TYPE; + let resolved = resolve_warehouse(warehouse, client).await?; let workspace_target = resolved.target; let value = resolved.value; + // From the resource's TYPE, not its shape: both kinds are objects with a `type` + // (Windmill's bigquery resource is a service-account JSON), so the value cannot + // say which it is. + let is_dbt_profile = resolved.resource_type == DBT_PROFILE_RESOURCE_TYPE; + // The block is written for the adapter it names, so a descriptor claiming another + // is a mistake worth naming rather than a profile rendered under the wrong type. + let stated = is_dbt_profile + .then(|| DbtAdapter::stated_by_dbt_profile(&value)) + .transpose()?; + if let (Some(declared), Some(stated)) = (declared.as_ref(), stated.as_ref()) { + if declared != stated { + return Err(Error::BadRequest(format!( + "`profile.type: {}` disagrees with the `{warehouse}` warehouse, whose resource \ + states `{}`. Remove `profile.type` or correct it", + declared.name(), + stated.name(), + ))); + } + } let adapter = declared - .or_else(|| DbtAdapter::infer_from_resource(&value)) + .or(stated) + // The resource TYPE: decision 9's authority, which the warehouse now carries. + // Inference reads connection details and covers a workspace's own type. + .or_else(|| KnownAdapter::from_resource_type(&resolved.resource_type).map(DbtAdapter::from)) + .or_else(|| KnownAdapter::infer_from_resource(&value).map(DbtAdapter::from)) .ok_or_else(|| { Error::BadRequest(format!( "could not tell which dbt adapter the `{warehouse}` warehouse needs; \ set `profile.type` in the descriptor" )) })?; - ensure_adapter_licensed(adapter)?; + ensure_adapter_licensed(&adapter)?; let profile_name = project_profile_name(project_dir, template_env).await; // The workspace's warehouse may name the target too, so a project that carries // no connection still gets `{{ target }}` right. @@ -1650,15 +1669,34 @@ async fn write_profiles( tokio::fs::create_dir_all(&dir) .await .map_err(|e| Error::internal_err(format!("creating the profiles dir: {e}")))?; - let rendered = render_profile( - adapter, - &value, - &profile_name, - target, - descriptor.threads, - descriptor.profile.schema.as_deref(), - &dir, - )?; + let rendered = if is_dbt_profile { + let block = value.as_object().ok_or_else(|| { + Error::BadRequest( + "a `dbt_profile` resource is a `profiles.yml` output block, so its value must be \ + an object" + .to_string(), + ) + })?; + render_dbt_profile( + &adapter, + block, + &profile_name, + target, + descriptor.threads, + descriptor.profile.schema.as_deref(), + &dir, + )? + } else { + render_profile( + &adapter, + &value, + &profile_name, + target, + descriptor.threads, + descriptor.profile.schema.as_deref(), + &dir, + )? + }; write_file(dir.to_str().unwrap(), "profiles.yml", &rendered.yaml)?; if let Some(pem) = rendered.root_certificate_pem.as_deref() { write_file( @@ -1800,8 +1838,7 @@ async fn adapter_from_profiles_yml( .and_then(|t| t.as_str()) .filter(|t| !t.contains("{{")); let adapter = match declared_type { - Some(t) => DbtAdapter::from_resource_type(t) - .ok_or_else(|| Error::BadRequest(format!("unsupported dbt adapter `{t}`")))?, + Some(t) => DbtAdapter::from_dbt_type(t)?, // REFUSED, not guessed. dbt renders the template and Windmill does not, // so the descriptor's word is the only thing left — and it is worth // nothing here: `profile.type: postgres` over a target resolving to @@ -5152,7 +5189,7 @@ mod tests { let t = adapter_from_profiles_yml(&path, "jaffle", None) .await .unwrap(); - assert_eq!(t.adapter, DbtAdapter::Snowflake); + assert_eq!(t.adapter, DbtAdapter::from(KnownAdapter::Snowflake)); assert_eq!(t.database.as_deref(), Some("prod")); assert_eq!(t.schema.as_deref(), Some("analytics")); // Spelled plainly, exactly as a rendered profile on the same relation. @@ -5185,7 +5222,7 @@ mod tests { let t = adapter_from_profiles_yml(&path, "jaffle", None) .await .unwrap(); - assert_eq!(t.adapter, DbtAdapter::Snowflake); + assert_eq!(t.adapter, DbtAdapter::from(KnownAdapter::Snowflake)); assert_eq!(t.database.as_deref(), Some("prod")); } diff --git a/backend/windmill-worker/src/dbt_profiles.rs b/backend/windmill-worker/src/dbt_profiles.rs index 03bc24ff22..3d96f84676 100644 --- a/backend/windmill-worker/src/dbt_profiles.rs +++ b/backend/windmill-worker/src/dbt_profiles.rs @@ -43,10 +43,12 @@ impl AdapterSpec { }; } -/// The dbt adapter a Windmill resource type maps to (decision 9). The resource -/// type name is the authority — connection details are never sniffed. +/// The adapters Windmill has facts about (decision 9): a field mapping, a pip package, the +/// license gate. NOT the adapters dbt projects may use — `DbtAdapter` carries those by name. +/// The picker offers what reaches one (`WAREHOUSE_RESOURCE_TYPES` in +/// `frontend/.../workspaceSettings/DbtSettings.svelte`), so a mapping change belongs there. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DbtAdapter { +pub enum KnownAdapter { Postgres, Redshift, Mysql, @@ -60,20 +62,41 @@ pub enum DbtAdapter { OracleDB, } -impl DbtAdapter { +impl KnownAdapter { + /// dbt's OWN `type:` spelling — a separate vocabulary from the resource types + /// below, deliberately. `fabric` is why: Windmill's `fabric` RESOURCE is a SQL + /// Server one, while dbt's `fabric` ADAPTER is its own `dbt-fabric`. Absent + /// here, it falls through to the open path and gets what it asked for. + pub fn from_dbt_type(t: &str) -> Option { + match t { + "postgres" | "postgresql" => Some(KnownAdapter::Postgres), + "redshift" => Some(KnownAdapter::Redshift), + "mysql" => Some(KnownAdapter::Mysql), + "duckdb" => Some(KnownAdapter::Duckdb), + "clickhouse" => Some(KnownAdapter::Clickhouse), + "snowflake" => Some(KnownAdapter::Snowflake), + "bigquery" => Some(KnownAdapter::Bigquery), + "databricks" => Some(KnownAdapter::Databricks), + "salesforce" => Some(KnownAdapter::Salesforce), + "sqlserver" | "mssql" => Some(KnownAdapter::Mssql), + "oracle" => Some(KnownAdapter::OracleDB), + _ => None, + } + } + pub fn from_resource_type(rt: &str) -> Option { match rt { - "postgresql" | "postgres" => Some(DbtAdapter::Postgres), - "redshift" => Some(DbtAdapter::Redshift), - "mysql" => Some(DbtAdapter::Mysql), - "duckdb" => Some(DbtAdapter::Duckdb), - "clickhouse" => Some(DbtAdapter::Clickhouse), - "snowflake" | "snowflake_oauth" => Some(DbtAdapter::Snowflake), - "bigquery" | "gcp_service_account" => Some(DbtAdapter::Bigquery), - "databricks" => Some(DbtAdapter::Databricks), - "salesforce" => Some(DbtAdapter::Salesforce), - "ms_sql_server" | "mssql" | "sqlserver" | "fabric" => Some(DbtAdapter::Mssql), - "oracledb" | "oracle" => Some(DbtAdapter::OracleDB), + "postgresql" | "postgres" => Some(KnownAdapter::Postgres), + "redshift" => Some(KnownAdapter::Redshift), + "mysql" => Some(KnownAdapter::Mysql), + "duckdb" => Some(KnownAdapter::Duckdb), + "clickhouse" => Some(KnownAdapter::Clickhouse), + "snowflake" | "snowflake_oauth" => Some(KnownAdapter::Snowflake), + "bigquery" | "gcp_service_account" => Some(KnownAdapter::Bigquery), + "databricks" => Some(KnownAdapter::Databricks), + "salesforce" => Some(KnownAdapter::Salesforce), + "ms_sql_server" | "mssql" | "sqlserver" | "fabric" => Some(KnownAdapter::Mssql), + "oracledb" | "oracle" => Some(KnownAdapter::OracleDB), _ => None, } } @@ -86,15 +109,15 @@ impl DbtAdapter { /// `dbname` instead of failing to build. fn spec(&self) -> &'static AdapterSpec { match self { - DbtAdapter::Postgres => &AdapterSpec::PG, - DbtAdapter::Redshift => &AdapterSpec { + KnownAdapter::Postgres => &AdapterSpec::PG, + KnownAdapter::Redshift => &AdapterSpec { name: "redshift", dbt_type: "redshift", pip_package: "dbt-redshift", default_port: 5439, ..AdapterSpec::PG }, - DbtAdapter::Mysql => &AdapterSpec { + KnownAdapter::Mysql => &AdapterSpec { name: "mysql", dbt_type: "mysql", pip_package: "dbt-mysql", @@ -102,31 +125,31 @@ impl DbtAdapter { database_key: "schema", ..AdapterSpec::PG }, - DbtAdapter::Duckdb => &AdapterSpec { + KnownAdapter::Duckdb => &AdapterSpec { name: "duckdb", dbt_type: "duckdb", pip_package: "dbt-duckdb", ..AdapterSpec::PG }, - DbtAdapter::Clickhouse => &AdapterSpec { + KnownAdapter::Clickhouse => &AdapterSpec { name: "clickhouse", dbt_type: "clickhouse", pip_package: "dbt-clickhouse", ..AdapterSpec::PG }, - DbtAdapter::Snowflake => &AdapterSpec { + KnownAdapter::Snowflake => &AdapterSpec { name: "snowflake", dbt_type: "snowflake", pip_package: "dbt-snowflake", ..AdapterSpec::PG }, - DbtAdapter::Bigquery => &AdapterSpec { + KnownAdapter::Bigquery => &AdapterSpec { name: "bigquery", dbt_type: "bigquery", pip_package: "dbt-bigquery", ..AdapterSpec::PG }, - DbtAdapter::Databricks => &AdapterSpec { + KnownAdapter::Databricks => &AdapterSpec { name: "databricks", dbt_type: "databricks", pip_package: "dbt-databricks", @@ -136,13 +159,13 @@ impl DbtAdapter { // `provision_core_1x` refuses it by name rather than asking uv to // install `""`. Pinned by // `every_adapter_either_names_a_package_or_is_fusion_only`. - DbtAdapter::Salesforce => &AdapterSpec { + KnownAdapter::Salesforce => &AdapterSpec { name: "salesforce", dbt_type: "salesforce", pip_package: "", ..AdapterSpec::PG }, - DbtAdapter::Mssql => &AdapterSpec { + KnownAdapter::Mssql => &AdapterSpec { name: "mssql", dbt_type: "sqlserver", pip_package: "dbt-sqlserver", @@ -150,7 +173,7 @@ impl DbtAdapter { display_name: Some("Microsoft SQL server"), ..AdapterSpec::PG }, - DbtAdapter::OracleDB => &AdapterSpec { + KnownAdapter::OracleDB => &AdapterSpec { name: "oracle", dbt_type: "oracle", pip_package: "dbt-oracle", @@ -165,18 +188,18 @@ impl DbtAdapter { /// there is: a second would be the thing that goes stale. Test-only, so a /// release build does not carry a table nothing reads. #[cfg(test)] - pub const ALL: &'static [DbtAdapter] = &[ - DbtAdapter::Postgres, - DbtAdapter::Redshift, - DbtAdapter::Mysql, - DbtAdapter::Duckdb, - DbtAdapter::Clickhouse, - DbtAdapter::Snowflake, - DbtAdapter::Bigquery, - DbtAdapter::Databricks, - DbtAdapter::Salesforce, - DbtAdapter::Mssql, - DbtAdapter::OracleDB, + pub const ALL: &'static [KnownAdapter] = &[ + KnownAdapter::Postgres, + KnownAdapter::Redshift, + KnownAdapter::Mysql, + KnownAdapter::Duckdb, + KnownAdapter::Clickhouse, + KnownAdapter::Snowflake, + KnownAdapter::Bigquery, + KnownAdapter::Databricks, + KnownAdapter::Salesforce, + KnownAdapter::Mssql, + KnownAdapter::OracleDB, ]; /// Which dbt driver a resource needs, from the fields it carries — a @@ -195,13 +218,13 @@ impl DbtAdapter { pub fn infer_from_resource(v: &Value) -> Option { let has = |k: &str| v.get(k).is_some_and(|x| !x.is_null()); if has("account_identifier") || has("warehouse") { - Some(DbtAdapter::Snowflake) + Some(KnownAdapter::Snowflake) } else if has("http_path") { - Some(DbtAdapter::Databricks) + Some(KnownAdapter::Databricks) } else if has("project_id") && has("client_email") { - Some(DbtAdapter::Bigquery) + Some(KnownAdapter::Bigquery) } else if has("dbname") && has("host") && (has("sslmode") || has("root_certificate_pem")) { - Some(DbtAdapter::Postgres) + Some(KnownAdapter::Postgres) } else { None } @@ -245,10 +268,16 @@ impl DbtAdapter { /// the same table plainly, and the two never share a node. pub fn target_identity_keys(&self) -> (&'static str, &'static str) { match self { - DbtAdapter::Snowflake => ("database", "schema"), - DbtAdapter::Bigquery => ("project", "dataset"), - DbtAdapter::Databricks => ("catalog", "schema"), - _ => (self.database_key(), "schema"), + KnownAdapter::Bigquery => ("project", "dataset"), + KnownAdapter::Databricks => ("catalog", "schema"), + // `database_key` is what Windmill's own resource spells it, and only + // the adapters `render_profile` translates have one. The rest reach + // dbt through a block they wrote themselves, which spells it the way + // dbt does. + KnownAdapter::Postgres | KnownAdapter::Redshift | KnownAdapter::Mysql => { + (self.database_key(), "schema") + } + _ => ("database", "schema"), } } @@ -270,6 +299,106 @@ impl DbtAdapter { } } +/// The adapter a profile connects with: dbt's own `type:`, plus whatever Windmill knows. +/// Open by construction — a `dbt_profile` carries a block Windmill never has to understand, +/// so an unknown adapter is still rendered, licensed and identified. INSTALLING one is a +/// separate question, gated by `ensure_adapter_installable`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DbtAdapter { + known: Option, + /// dbt's own `type:` spelling, lowercased. Also the pip package's suffix. + name: String, +} + +impl DbtAdapter { + /// An adapter as dbt spells it, known or not. The name reaches a pip requirement + /// and a venv path, so it is confined to what an adapter name can be rather than + /// escaped at each use: a leading `-` is a pip flag, a `/` or `..` a path segment. + pub fn from_dbt_type(t: &str) -> error::Result { + let name = t.trim().to_ascii_lowercase(); + let shaped = name.len() <= 40 + && name.starts_with(|c: char| c.is_ascii_alphanumeric()) + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-'); + if !shaped { + return Err(Error::BadRequest(format!( + "`{t}` is not a dbt adapter name: dbt spells one with letters, digits, `_` and \ + `-`, as in `postgres` or `trino`" + ))); + } + // Normalised to the adapter's own dbt spelling when one resolves, so two + // names for one adapter are one value: `PartialEq` covers `name` too, and + // `profile.type: mssql` over a `sqlserver` target would otherwise be + // rejected by a message naming the same adapter on both sides. + let known = KnownAdapter::from_dbt_type(&name); + let name = known.map_or(name, |k| k.dbt_type().to_string()); + Ok(Self { known, name }) + } + + /// The adapter a `dbt_profile` states, from the `type` of the block its value is. + /// Called only for that resource TYPE, never sniffed: Windmill's bigquery resource + /// is a service-account JSON and says `type: service_account`. + pub fn stated_by_dbt_profile(v: &Value) -> error::Result { + let stated = v.get("type").and_then(|t| t.as_str()).ok_or_else(|| { + Error::BadRequest( + "a `dbt_profile` resource names its adapter in `type`, as a `profiles.yml` output \ + does; this one has none" + .to_string(), + ) + })?; + Self::from_dbt_type(stated) + } + + /// The adapter's facts, when it is one Windmill has any. + pub fn known(&self) -> Option { + self.known + } + + /// dbt's own `type:` key in `profiles.yml`. + pub fn dbt_type(&self) -> &str { + self.known.map_or(self.name.as_str(), |k| k.dbt_type()) + } + + /// The adapter's name as a user would write it, for error messages. + pub fn name(&self) -> &str { + self.known.map_or(self.name.as_str(), |k| k.name()) + } + + /// The pip package providing this adapter for the dbt-core 1.x engine. + /// Empty means no such package exists and only Fusion has it. + pub fn pip_package(&self) -> String { + match self.known { + Some(k) => k.pip_package().to_string(), + None => format!("dbt-{}", self.name), + } + } + + /// The keys a target block spells its database and schema with. + /// An unknown adapter is read with dbt's ordinary pair. + pub fn target_identity_keys(&self) -> (&'static str, &'static str) { + self.known + .map_or(("database", "schema"), |k| k.target_identity_keys()) + } + + /// Whether this adapter needs an enterprise license. Never for an unknown + /// one: the gate mirrors the two native warehouse languages, and an adapter + /// with no Windmill runtime behind it is not one of them. + pub fn requires_enterprise(&self) -> bool { + self.known.is_some_and(|k| k.requires_enterprise()) + } + + fn display_name(&self) -> &str { + self.known.map_or(self.name.as_str(), |k| k.display_name()) + } +} + +impl From for DbtAdapter { + fn from(known: KnownAdapter) -> Self { + Self { known: Some(known), name: known.dbt_type().to_string() } + } +} + /// Whether this build may use the enterprise-only adapters. /// /// Two conditions, both required. `LICENSE_KEY_VALID` alone is not enough: the @@ -294,7 +423,7 @@ fn enterprise_licensed() -> bool { /// one dbt executor and the adapter is only known once the profile resolves. So /// it is checked at both deploy and run — and it says so plainly, rather than /// letting the run fail later with a connection error the user cannot act on. -pub fn ensure_adapter_licensed(adapter: DbtAdapter) -> error::Result<()> { +pub fn ensure_adapter_licensed(adapter: &DbtAdapter) -> error::Result<()> { if adapter.requires_enterprise() && !enterprise_licensed() { return Err(Error::BadRequest(format!( "{} is only available with an enterprise license", @@ -355,7 +484,7 @@ pub struct RenderedProfile { /// per-adapter default — dbt errors out clearly when it ends up missing, which /// is a better failure than a Windmill-invented default. pub fn render_profile( - adapter: DbtAdapter, + adapter: &DbtAdapter, resource: &Value, profile_name: &str, target: &str, @@ -367,6 +496,17 @@ pub fn render_profile( // resolved against the project and never found. profiles_dir: &std::path::Path, ) -> error::Result { + // Past here the resource is one of Windmill's own connection types, which + // becomes a target only through a field mapping below — so an adapter + // Windmill has no facts about cannot be rendered from one at all. + let adapter = adapter.known().ok_or_else(|| { + Error::BadRequest(format!( + "no Windmill resource translates into a `{}` target; point the warehouse at a \ + `dbt_profile` resource, whose value is the profiles.yml block itself", + adapter.dbt_type() + )) + })?; + let mut out: Vec<(String, ProfileValue)> = vec![("type".into(), quoted(adapter.dbt_type()))]; let mut schema = schema_override.map(|x| x.to_string()); let database; @@ -376,7 +516,7 @@ pub fn render_profile( // shape as Postgres in both dbt and Windmill's resource types, so one // arm renders all three; only the default port and the database key // differ. - DbtAdapter::Postgres | DbtAdapter::Redshift | DbtAdapter::Mysql => { + KnownAdapter::Postgres | KnownAdapter::Redshift | KnownAdapter::Mysql => { let host = s(resource, "host").ok_or_else(|| { Error::BadRequest(format!("{} resource has no `host`", adapter.dbt_type())) })?; @@ -420,7 +560,7 @@ pub fn render_profile( schema = match adapter { // Already emitted as the database key; reported back so the // caller can spell `dbt://` paths with it. - DbtAdapter::Mysql => Some(dbname.clone()), + KnownAdapter::Mysql => Some(dbname.clone()), _ => schema .or_else(|| s(resource, "schema")) .or(Some("public".into())), @@ -428,20 +568,21 @@ pub fn render_profile( } // Their Windmill resources do not carry what dbt needs — an `oracledb` // resource has no host/service, dbt-sqlserver needs an ODBC `driver` the - // images lack — so a rendered profile could not connect. They go through - // the project's own `profiles.yml` instead. - DbtAdapter::Duckdb - | DbtAdapter::Clickhouse - | DbtAdapter::Salesforce - | DbtAdapter::Mssql - | DbtAdapter::OracleDB => { + // images lack — so a rendered profile could not connect. They are reached + // through a target written for them instead. + KnownAdapter::Duckdb + | KnownAdapter::Clickhouse + | KnownAdapter::Salesforce + | KnownAdapter::Mssql + | KnownAdapter::OracleDB => { return Err(Error::BadRequest(format!( - "the `{}` adapter has no Windmill resource mapping; point \ - `profile.profiles_yml` at the project's own profiles.yml instead", + "a `{}` resource carries nothing dbt can connect with; point the warehouse at a \ + `dbt_profile` resource, whose value is the profiles.yml block itself, or \ + `profile.profiles_yml` at the project's own profiles.yml", adapter.dbt_type() ))); } - DbtAdapter::Snowflake => { + KnownAdapter::Snowflake => { let account = s(resource, "account_identifier") .or_else(|| s(resource, "account")) .ok_or_else(|| { @@ -477,7 +618,7 @@ pub fn render_profile( } schema = schema.or_else(|| s(resource, "schema")); } - DbtAdapter::Bigquery => { + KnownAdapter::Bigquery => { // Windmill's bigquery resource is the raw service-account JSON, so // hand dbt the same document via `method: service-account-json` // rather than re-deriving individual fields. @@ -501,12 +642,18 @@ pub fn render_profile( )); } } - DbtAdapter::Databricks => { - for (k, rk) in [ - ("host", "host"), - ("http_path", "http_path"), - ("token", "token"), - ] { + KnownAdapter::Databricks => { + // Windmill's databricks resource spells the workspace `workspace_url`, and + // spells it as a full URL; dbt wants a bare hostname under `host`. + let host = s(resource, "host") + .or_else(|| s(resource, "workspace_url").map(|u| bare_host(&u))) + .ok_or_else(|| { + Error::BadRequest( + "databricks resource has no `workspace_url`/`host`".to_string(), + ) + })?; + out.push(("host".into(), quoted(&host))); + for (k, rk) in [("http_path", "http_path"), ("token", "token")] { let v = s(resource, rk).ok_or_else(|| { Error::BadRequest(format!("databricks resource has no `{rk}`")) })?; @@ -524,12 +671,12 @@ pub fn render_profile( // database, already emitted above. Pushing the generic one too would put // two `schema` keys in one target — an invalid profile, or one silently // pointing at the wrong database. - if adapter != DbtAdapter::Mysql { + if adapter != KnownAdapter::Mysql { if let Some(sc) = schema.clone() { // dbt-bigquery spells it `dataset`; every other adapter says // `schema`. Emitting `schema` there produces a profile dbt rejects. let key = match adapter { - DbtAdapter::Bigquery => "dataset", + KnownAdapter::Bigquery => "dataset", _ => "schema", }; out.push((key.into(), quoted(&sc))); @@ -549,7 +696,7 @@ pub fn render_profile( yaml.push_str(&format!(" {k}: {}\n", v.render())); } // The service-account document is a nested mapping, not a scalar. - if adapter == DbtAdapter::Bigquery { + if adapter == KnownAdapter::Bigquery { yaml.push_str(" keyfile_json:\n"); let obj = resource .as_object() @@ -565,12 +712,141 @@ pub fn render_profile( yaml, schema, database, - root_certificate_pem: matches!(adapter, DbtAdapter::Postgres) + root_certificate_pem: matches!(adapter, KnownAdapter::Postgres) .then(|| s(resource, "root_certificate_pem")) .flatten(), }) } +/// Render a `dbt_profile`: its value IS one entry of `profiles.yml`'s `outputs` map, so it +/// is emitted as it stands — nothing lifted out or renamed, which is the point of the type. +/// Only what dbt cannot take literally is handled: the adapter's `type`, a certificate that +/// is a PEM body rather than a path, and the two keys a descriptor may override. +pub fn render_dbt_profile( + adapter: &DbtAdapter, + block: &serde_json::Map, + profile_name: &str, + target: &str, + threads: Option, + schema_override: Option<&str>, + profiles_dir: &std::path::Path, +) -> error::Result { + let (database_key, schema_key) = adapter.target_identity_keys(); + let root_certificate_pem = block + .get("root_certificate_pem") + .and_then(|v| v.as_str()) + .filter(|v| !v.is_empty()) + .map(|v| v.to_string()); + + let (qp, qt) = (yaml_scalar(profile_name), yaml_scalar(target)); + let mut yaml = format!("{qp}:\n target: {qt}\n outputs:\n {qt}:\n"); + // Keys quoted throughout this block, the block's own and Windmill's alike: a + // target whose `"host"` is quoted and whose `schema` is not reads as a bug. + yaml.push_str(&format!( + " \"type\": {}\n", + yaml_scalar(adapter.dbt_type()) + )); + for (k, v) in block { + // A null is an optional field the resource form left unset, and dbt + // validates several keys against a schema that rejects one. + if k == "type" || k == "root_certificate_pem" || v.is_null() { + continue; + } + // Only when Windmill writes one of its own, which would otherwise be a second + // `sslrootcert`. A path with no PEM beside it is the block's own trust source — + // a CA baked into the image or mounted on the worker — and dropping it changes + // what the connection verifies against. + if k == "sslrootcert" && root_certificate_pem.is_some() { + continue; + } + if (k == schema_key && schema_override.is_some()) || (k == "threads" && threads.is_some()) { + continue; + } + emit_entry(&mut yaml, 6, k, v); + } + if root_certificate_pem.is_some() { + yaml.push_str(&format!( + " \"sslrootcert\": {}\n", + yaml_scalar(&profiles_dir.join(ROOT_CERT_FILENAME).to_string_lossy()) + )); + } + if let Some(sc) = schema_override { + yaml.push_str(&format!( + " {}: {}\n", + yaml_scalar(schema_key), + yaml_scalar(sc) + )); + } + if let Some(t) = threads { + yaml.push_str(&format!(" \"threads\": {t}\n")); + } + + let str_key = |k: &str| block.get(k).and_then(|v| v.as_str()).map(|v| v.to_string()); + Ok(RenderedProfile { + yaml, + schema: schema_override + .map(|x| x.to_string()) + .or_else(|| str_key(schema_key)), + database: str_key(database_key), + root_certificate_pem, + }) +} + +/// Emit one target key, nesting as deep as the value goes — an adapter's credential can be +/// a mapping (bigquery's `keyfile_json`) or a list. Keys are quoted like values: one nothing +/// here enumerates is as free-form as a password. +fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value) { + out.push_str(&format!("{}{}:", " ".repeat(indent), yaml_scalar(key))); + emit_value(out, indent, v); +} + +/// The value half, after `key:`. An empty collection is emitted INLINE: a block with no +/// children reads back as `null`, so `extensions: []` would reach the adapter as a missing +/// value rather than the empty list dbt was handed. +fn emit_value(out: &mut String, indent: usize, v: &Value) { + match v { + Value::Object(m) => { + // A null is an optional field the resource form left unset, and dbt validates + // several keys against a schema that rejects one. + let kept: Vec<_> = m.iter().filter(|(_, v)| !v.is_null()).collect(); + if kept.is_empty() { + out.push_str(" {}\n"); + return; + } + out.push('\n'); + for (k, v) in kept { + emit_entry(out, indent + 2, k, v); + } + } + Value::Array(items) => { + if items.is_empty() { + out.push_str(" []\n"); + return; + } + out.push('\n'); + let pad = " ".repeat(indent + 2); + for item in items { + out.push_str(&pad); + out.push('-'); + emit_value(out, indent + 2, item); + } + } + _ => out.push_str(&format!(" {}\n", yaml_value(v))), + } +} + +/// A scalar as dbt reads it: quoted when it is text, bare when it is not. +/// `port`, `threads` and the boolean toggles adapters carry are validated +/// against a JSON schema that rejects the quoted form. +fn yaml_value(v: &Value) -> String { + match v { + Value::String(s) => yaml_scalar(s), + Value::Bool(b) => b.to_string(), + Value::Number(n) => n.to_string(), + other => yaml_scalar(&other.to_string()), + } +} + /// One rendered `profiles.yml` value. Strings are always quoted so credential /// content cannot inject profile keys; numbers must NOT be, because dbt /// validates `port` and `threads` against a JSON schema that demands integers @@ -593,6 +869,17 @@ fn quoted(v: &str) -> ProfileValue { ProfileValue::Str(v.to_string()) } +/// A workspace URL as dbt wants it: the hostname alone. Windmill's databricks +/// resource carries the full deployment URL, while dbt-databricks builds its own +/// URL from `host`, so a scheme left in place produces `https://https://…`. +fn bare_host(url: &str) -> String { + url.trim() + .trim_start_matches("https://") + .trim_start_matches("http://") + .trim_end_matches('/') + .to_string() +} + /// Emit a YAML scalar that survives any credential content. Always /// double-quoted with the two characters that can terminate or escape inside a /// double-quoted scalar escaped, so a password containing `#`, `:`, a newline @@ -616,7 +903,7 @@ mod tests { let r = json!({"host": "db.internal", "port": 5433, "user": "u", "password": "p", "dbname": "warehouse", "sslmode": "require"}); let p = render_profile( - DbtAdapter::Postgres, + &KnownAdapter::Postgres.into(), &r, "wm", "prod", @@ -648,7 +935,7 @@ mod tests { let r = json!({"host": "cluster.redshift.amazonaws.com", "user": "u", "password": "p", "dbname": "warehouse"}); let p = render_profile( - DbtAdapter::Redshift, + &KnownAdapter::Redshift.into(), &r, "wm", "prod", @@ -660,6 +947,211 @@ mod tests { assert!(p.yaml.contains(" port: 5439\n"), "{}", p.yaml); } + // The point of `dbt_profile`: an adapter with no field mapping still + // connects, because its value is a block dbt wrote and reaches dbt as it is. + #[test] + fn dbt_profile_renders_its_block_verbatim() { + let r = json!({"type": "clickhouse", "host": "ch.internal", "port": 8123, + "secure": true, "user": "u", "password": "p", "schema": "analytics"}); + let p = render_dbt_profile( + &KnownAdapter::Clickhouse.into(), + r.as_object().unwrap(), + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap(); + assert!( + p.yaml.contains(" \"type\": \"clickhouse\"\n"), + "{}", + p.yaml + ); + assert!( + p.yaml.contains(" \"host\": \"ch.internal\"\n"), + "{}", + p.yaml + ); + // dbt types these in its own schema and rejects the quoted form. + assert!(p.yaml.contains(" \"port\": 8123\n"), "{}", p.yaml); + assert!(p.yaml.contains(" \"secure\": true\n"), "{}", p.yaml); + // The identity a `dbt://` path is spelled with, read back from the block. + assert_eq!(p.schema.as_deref(), Some("analytics")); + } + + // Two `schema` keys in one target is a profile dbt rejects, or one silently + // pointing at the schema the descriptor meant to override. + #[test] + fn dbt_profile_schema_override_replaces_the_block_key() { + let r = json!({"type": "clickhouse", "host": "h", "schema": "raw"}); + let p = render_dbt_profile( + &KnownAdapter::Clickhouse.into(), + r.as_object().unwrap(), + "wm", + "prod", + None, + Some("staging"), + std::path::Path::new("/tmp/p"), + ) + .unwrap(); + assert_eq!(p.yaml.matches("\"schema\":").count(), 1, "{}", p.yaml); + assert!(p.yaml.contains("\"schema\": \"staging\"\n"), "{}", p.yaml); + assert_eq!(p.schema.as_deref(), Some("staging")); + } + + // `fabric` is a distinct dbt adapter AND a Windmill resource type for SQL + // Server. Resolving dbt's `type:` through the resource table installed + // dbt-sqlserver for it, under an enterprise gate, and never said Fabric. + #[test] + fn a_dbt_type_is_not_a_resource_type() { + let fabric = DbtAdapter::from_dbt_type("fabric").unwrap(); + assert_eq!(fabric.dbt_type(), "fabric"); + assert_eq!(fabric.pip_package(), "dbt-fabric"); + assert!(!fabric.requires_enterprise()); + // The Windmill resource type keeps mapping where it always did. + assert_eq!( + KnownAdapter::from_resource_type("fabric"), + Some(KnownAdapter::Mssql) + ); + } + + // `PartialEq` covers the carried name, so two spellings of one adapter must + // normalise or the descriptor/resource agreement check rejects a valid + // config with a message naming the same adapter on both sides. + #[test] + fn two_spellings_of_one_adapter_are_one_value() { + for (a, b) in [("postgres", "postgresql"), ("sqlserver", "mssql")] { + assert_eq!( + DbtAdapter::from_dbt_type(a).unwrap(), + DbtAdapter::from_dbt_type(b).unwrap(), + "{a} vs {b}" + ); + } + } + + // An adapter Windmill has no facts about is still an adapter: this is what + // "whatever dbt supports" rests on. + #[test] + fn an_unknown_adapter_is_carried_by_name() { + let stated = DbtAdapter::stated_by_dbt_profile(&json!({"type": "trino", "host": "h"})) + .unwrap(); + assert_eq!(stated.dbt_type(), "trino"); + assert_eq!(stated.pip_package(), "dbt-trino"); + assert!(stated.known().is_none()); + // A block with no adapter cannot be rendered, and says which key is missing. + assert!(DbtAdapter::stated_by_dbt_profile(&json!({"host": "h"})).is_err()); + } + + // A block whose CA is a path on the worker keeps it: Windmill only takes the + // key over when it has a PEM of its own to point at. + #[test] + fn a_path_only_sslrootcert_survives() { + let r = json!({"type": "postgres", "host": "h", "sslmode": "verify-full", + "sslrootcert": "/etc/ssl/certs/warehouse-ca.pem"}); + let p = render_dbt_profile( + &KnownAdapter::Postgres.into(), + r.as_object().unwrap(), + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap(); + assert!( + p.yaml + .contains("\"sslrootcert\": \"/etc/ssl/certs/warehouse-ca.pem\"\n"), + "{}", + p.yaml + ); + // And a PEM in the block still wins, exactly once. + let r = json!({"type": "postgres", "host": "h", "sslrootcert": "/ignored", + "root_certificate_pem": "-----BEGIN CERTIFICATE-----\nx\n"}); + let p = render_dbt_profile( + &KnownAdapter::Postgres.into(), + r.as_object().unwrap(), + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap(); + assert_eq!(p.yaml.matches("sslrootcert").count(), 1, "{}", p.yaml); + assert!(p.yaml.contains(ROOT_CERT_FILENAME), "{}", p.yaml); + } + + // dbt hands these to the adapter as it finds them, so a collection must survive the + // round trip: a block with no children reads back as `null`, not as `[]` or `{}`. + #[test] + fn collections_keep_their_type() { + let r = json!({"type": "duckdb", "extensions": [], "settings": {}, + "attach": [{"path": "raw.db", "read_only": true}], + "matrix": [["a", 1], []], "plugins": ["excel", "json"]}); + let p = render_dbt_profile( + &DbtAdapter::from_dbt_type("duckdb").unwrap(), + r.as_object().unwrap(), + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap(); + // Parsed back rather than string-matched: the point is what a YAML reader sees. + let y: serde_json::Value = serde_yml::from_str(&p.yaml).expect(&p.yaml); + let t = &y["wm"]["outputs"]["prod"]; + assert_eq!(t["extensions"], json!([]), "{}", p.yaml); + assert_eq!(t["settings"], json!({}), "{}", p.yaml); + assert_eq!(t["attach"], json!([{"path": "raw.db", "read_only": true}]), "{}", p.yaml); + assert_eq!(t["matrix"], json!([["a", 1], []]), "{}", p.yaml); + assert_eq!(t["plugins"], json!(["excel", "json"]), "{}", p.yaml); + } + + // An unknown adapter's name becomes `dbt-` in a pip requirement and a + // venv path, both on the host and outside the jail. + #[test] + fn an_adapter_name_cannot_be_a_pip_flag_or_a_path() { + for bad in [ + "--index-url=http://evil", + "-e .", + "../../etc/passwd", + "dbt postgres", + "", + ] { + assert!(DbtAdapter::from_dbt_type(bad).is_err(), "{bad}"); + } + assert_eq!( + DbtAdapter::from_dbt_type("TRINO").unwrap().dbt_type(), + "trino" + ); + } + + // Windmill's databricks resource spells the workspace as a full URL under + // `workspace_url`; dbt-databricks builds its own URL from a bare `host`. + #[test] + fn databricks_takes_its_host_from_the_workspace_url() { + let r = json!({"workspace_url": "https://dbc-a1b2.cloud.databricks.com/", + "http_path": "/sql/1.0/warehouses/x", "token": "t"}); + let p = render_profile( + &KnownAdapter::Databricks.into(), + &r, + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap(); + assert!( + p.yaml + .contains(" host: \"dbc-a1b2.cloud.databricks.com\"\n"), + "{}", + p.yaml + ); + } + // A resource's private CA is the only way a `verify-full` connection can // succeed, and `root_certificate_pem` is also what identifies the resource as // Postgres — forwarding one and dropping the other is incoherent. @@ -668,7 +1160,7 @@ mod tests { let r = json!({"host": "h", "dbname": "d", "sslmode": "verify-full", "root_certificate_pem": "-----BEGIN CERTIFICATE-----\nx\n"}); let p = render_profile( - DbtAdapter::Postgres, + &KnownAdapter::Postgres.into(), &r, "wm", "prod", @@ -695,7 +1187,7 @@ mod tests { // No CA configured, no dangling sslrootcert pointing at a missing file. let plain = json!({"host": "h", "dbname": "d", "sslmode": "require"}); let p = render_profile( - DbtAdapter::Postgres, + &KnownAdapter::Postgres.into(), &plain, "wm", "prod", @@ -716,7 +1208,7 @@ mod tests { let r = json!({"account_identifier": "acc", "username": "u", "token": "tok", "database": "db", "warehouse": "wh"}); let p = render_profile( - DbtAdapter::Snowflake, + &KnownAdapter::Snowflake.into(), &r, "wm", "prod", @@ -740,7 +1232,7 @@ mod tests { fn bigquery_requires_a_dataset() { let r = json!({"project_id": "p", "client_email": "e", "private_key": "k"}); let err = render_profile( - DbtAdapter::Bigquery, + &KnownAdapter::Bigquery.into(), &r, "wm", "prod", @@ -752,7 +1244,7 @@ mod tests { .to_string(); assert!(err.contains("profile.schema"), "{err}"); let p = render_profile( - DbtAdapter::Bigquery, + &KnownAdapter::Bigquery.into(), &r, "wm", "prod", @@ -772,16 +1264,16 @@ mod tests { #[test] fn ambiguous_host_resources_decline_rather_than_guess() { let mssql = json!({"host": "h", "dbname": "d", "user": "u", "password": "p"}); - assert_eq!(DbtAdapter::infer_from_resource(&mssql), None); + assert_eq!(KnownAdapter::infer_from_resource(&mssql), None); let pg = json!({"host": "h", "dbname": "d", "sslmode": "require"}); assert_eq!( - DbtAdapter::infer_from_resource(&pg), - Some(DbtAdapter::Postgres) + KnownAdapter::infer_from_resource(&pg), + Some(KnownAdapter::Postgres) ); let sf = json!({"account_identifier": "acc", "database": "d"}); assert_eq!( - DbtAdapter::infer_from_resource(&sf), - Some(DbtAdapter::Snowflake) + KnownAdapter::infer_from_resource(&sf), + Some(KnownAdapter::Snowflake) ); } @@ -791,7 +1283,7 @@ mod tests { fn mysql_emits_exactly_one_schema_key() { let r = json!({"host": "h", "dbname": "sales", "user": "u"}); let p = render_profile( - DbtAdapter::Mysql, + &KnownAdapter::Mysql.into(), &r, "wm", "dev", @@ -811,7 +1303,7 @@ mod tests { #[test] fn a_profile_name_or_target_cannot_open_a_sibling_key() { let rendered = render_profile( - DbtAdapter::Postgres, + &KnownAdapter::Postgres.into(), &serde_json::json!({"host": "h", "user": "u", "password": "p", "dbname": "d"}), "prod # hidden", "dev\n evil: yes", @@ -846,7 +1338,7 @@ mod tests { let r = json!({"host": "h", "dbname": "d", "password": "p\"\nhost: evil.example.com\n#"}); let p = render_profile( - DbtAdapter::Postgres, + &KnownAdapter::Postgres.into(), &r, "wm", "dev", @@ -872,25 +1364,25 @@ mod tests { #[test] fn only_mssql_and_oracle_are_enterprise_gated() { for a in [ - DbtAdapter::Postgres, - DbtAdapter::Redshift, - DbtAdapter::Mysql, - DbtAdapter::Duckdb, - DbtAdapter::Clickhouse, - DbtAdapter::Snowflake, - DbtAdapter::Bigquery, - DbtAdapter::Databricks, - DbtAdapter::Salesforce, + KnownAdapter::Postgres, + KnownAdapter::Redshift, + KnownAdapter::Mysql, + KnownAdapter::Duckdb, + KnownAdapter::Clickhouse, + KnownAdapter::Snowflake, + KnownAdapter::Bigquery, + KnownAdapter::Databricks, + KnownAdapter::Salesforce, ] { assert!(!a.requires_enterprise(), "{a:?}"); - assert!(ensure_adapter_licensed(a).is_ok(), "{a:?}"); + assert!(ensure_adapter_licensed(&a.into()).is_ok(), "{a:?}"); } for (a, name) in [ - (DbtAdapter::Mssql, "Microsoft SQL server"), - (DbtAdapter::OracleDB, "Oracle DB"), + (KnownAdapter::Mssql, "Microsoft SQL server"), + (KnownAdapter::OracleDB, "Oracle DB"), ] { assert!(a.requires_enterprise(), "{a:?}"); - match ensure_adapter_licensed(a) { + match ensure_adapter_licensed(&a.into()) { Ok(()) => assert!( enterprise_licensed(), "{a:?} was accepted without an enterprise license" diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index 62e680d3a1..dc06a22fa0 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -31,7 +31,7 @@ the dominant way dbt is orchestrated today. | 6 | Multiple run configs | Per-run `select` on one script; N scripts means N projects | | 7 | Run-time `select` | Descriptor default plus run-arg override | | 8 | Credentials | Workspace warehouses, plus `profiles.yml` passthrough. A descriptor never names a resource. See below | -| 9 | Adapter mappings | postgres, redshift, mysql, snowflake, bigquery, databricks; others via the project's own `profiles.yml` | +| 9 | Adapter mappings | postgres, redshift, mysql, snowflake, bigquery, databricks translate from their Windmill resource; **every** adapter dbt has is reachable from a `dbt_profile` resource, or the project's own `profiles.yml` | | 10 | Private repo auth | Not applicable: the project is synced, not fetched | | 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | @@ -122,11 +122,19 @@ salesforce) is CE. Gating any of the others would make reaching a warehouse through dbt stricter than reaching it natively, which is backwards. Those two are *recognized* (for the gate and for the pip package the 1.x -engine's venv needs), not rendered from a resource: an `oracledb` resource is -`{user, password, database}` with no host/protocol/service, and dbt-sqlserver -needs an ODBC `driver` the images do not install. Both reach their warehouse -through the project's own `profiles.yml`, which is also how duckdb, clickhouse -and salesforce work. +engine's venv needs), but no Windmill connection resource translates into them: +an `oracledb` resource is `{user, password, database}` with no +host/protocol/service, and dbt-sqlserver needs an ODBC `driver` the images do not +install. They reach their warehouse through a `dbt_profile` resource or the +project's own `profiles.yml`, which is also how duckdb, clickhouse and salesforce +work. + +Recognition is what the gate keys on, and it survives the open adapter set: a +`dbt_profile` stating `sqlserver`, `mssql` or `oracle` resolves to the same +`KnownAdapter` a resource type would, so it is gated identically. An adapter +Windmill has never heard of is never enterprise — the boundary mirrors the two +native warehouse languages, and an adapter with no Windmill runtime behind it is +not one of them. The gate almost never fires in practice: `dbt-core-2x` supports neither adapter, so it can only apply to `dbt-core-1x` with one of those two. @@ -220,6 +228,65 @@ none) and cannot name a resource. Admins configure the warehouses under Settings → dbt, where each entry points at a resource, exactly as `large_file_storage` points at the object-storage resource and a DuckLake names its catalog. +**What a warehouse may point at.** Either a Windmill connection resource whose +type `render_profile` translates (`postgresql`, `redshift`, `mysql`, `snowflake`, +`snowflake_oauth`, `bigquery`, `gcp_service_account`, `databricks`), or a +**`dbt_profile`** resource, whose VALUE IS one entry of that file's `outputs` +map — `type` included, nothing lifted out or renamed. A block is copied from a +working `profiles.yml` and pasted in, which is the whole point: a type that asked +the user to restructure their block first would be doing the translation this +exists to avoid. Its schema declares no properties, so the resource form renders +one JSON editor over the value (`ResourceForm.svelte`). The picker is +constrained to exactly these (`WAREHOUSE_RESOURCE_TYPES`); anything else has no +way to become a target at all, which is why an unconstrained picker was a trap: +it offered slack and github resources for a field that can only be a warehouse. + +The two exist for different reasons. A Windmill resource is the ergonomic path +and is shared with everything else that connects to that warehouse, but it is +*not* a dbt target: each adapter arm translates the fields Windmill's resource +happens to carry into the keys dbt reads, so only what an arm covers can be +expressed, and an adapter with no arm cannot be reached from one at all. +`dbt_profile` inverts that — nothing is translated, so any adapter and any key it +documents works. + +Which of the two a value is cannot be read off the value: both are objects with a +`type`, and Windmill's bigquery resource is a service-account JSON that says +`type: service_account`. So the warehouse carries its resource's TYPE +(`DbtWarehouseConnection.resource_type`), and that is also what finally makes +decision 9's "the resource type name is the authority" true at runtime rather +than aspirational — the translated path resolved its adapter by sniffing +connection fields until it had the name. + +**`dbt_profile` is open, deliberately.** Its `type` is not checked against a list: +`DbtAdapter` carries an optional `KnownAdapter` beside the name, so the eleven +adapters Windmill has facts about (a field mapping, a pip package, the license +gate) keep them, and every other adapter dbt has — `trino`, `athena`, `spark`, +whatever ships next — is carried by name and rendered, licensed and identified +without Windmill knowing anything about it. A closed list would have made +"whatever dbt supports" mean "whatever this enum lists", and each new adapter a +Windmill release. The name is constrained to `[a-z0-9_-]` starting alphanumeric +*because* it is open: it reaches a pip requirement and a venv path on the host, +where a leading `-` is a flag and a `/` is a path segment. + +**Installing one is a separate question from using one.** `dbt-core-1x` fetches +`dbt-` from PyPI, `dbt-` is not a reserved prefix there, and that install +runs through `run_tool` — outside the nsjail ordinary Python dependency +installation uses, with uv executing a source distribution's PEP 517 backend. An +unbounded name would therefore let a script author publish `dbt-` and run code +as the worker, on the one dependency path that is not sandboxed. So +`ensure_adapter_installable` gates that install on `PUBLISHED_ADAPTERS` plus +whatever an operator lists in `DBT_EXTRA_ADAPTERS`: the author chooses which +adapter to use, the admin decides which packages this instance trusts. Nothing +else is gated — a profile still renders for any adapter, and `dbt-core-2x` and +`fusion` carry their adapters in the binary, install nothing, and take any +`type` at all. + +Two keys are not passed through: `type` (Windmill writes the adapter's own dbt +spelling) and `root_certificate_pem`, which is a PEM body rather than the path +dbt hands the driver — it is written beside `profiles.yml` and pointed at by +`sslrootcert`, as it is for a translated postgres resource. `profile.schema` and +`threads` from the descriptor override their block keys rather than joining them. + Three things follow, and they are the reason for the rule rather than consequences to work around. diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index c0a25daeb7..e3927ddc64 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -58,7 +58,11 @@ viewJsonSchema = false try { schema = resourceTypeInfo.schema as any - schema.order = schema.order ?? Object.keys(schema.properties).sort() + // A resource type may declare no properties at all — `dbt_profile` is a + // `profiles.yml` block whose keys are its adapter's, not Windmill's. That + // is a JSON-edited type, NOT a missing one: `Object.keys(undefined)` threw + // into the catch below, so the drawer told the user to sync a type it had. + schema.order = schema.order ?? Object.keys(schema.properties ?? {}).sort() notFound = false } catch (e) { notFound = true @@ -233,7 +237,7 @@ > {/if} -{#if notFound || viewJsonSchema} +{#if notFound || viewJsonSchema || !schema?.properties} {#if !emptyString(error)}{error}{:else}
{/if} diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 7acc4e485b..8b10841512 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -130,7 +130,11 @@ const schema = rt.schema as Schema return { ...schema, - order: schema.order ?? Object.keys(schema.properties).sort() + // A resource type may declare no properties at all — `dbt_profile` is a + // `profiles.yml` block whose keys are its adapter's, not Windmill's — and + // the form renders those as one JSON editor. `Object.keys(undefined)` + // threw here, which left the editor on its loading skeleton forever. + order: schema.order ?? Object.keys(schema.properties ?? {}).sort() } }) let loadingSchema = $derived(resourceTypeResource.loading) diff --git a/frontend/src/lib/components/ResourceForm.svelte b/frontend/src/lib/components/ResourceForm.svelte index 2c563a8e95..3820004593 100644 --- a/frontend/src/lib/components/ResourceForm.svelte +++ b/frontend/src/lib/components/ResourceForm.svelte @@ -291,7 +291,7 @@ {:else if !can_write} {:else} - {#if !viewJsonSchema} + {#if !viewJsonSchema && !resourceSchema}

Resource type '{resource_type}' not found in your workspace diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index d6ac00a755..194d1392fe 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -275,6 +275,7 @@ export const APP_TO_ICON_COMPONENT = { appwrite: AppwriteIcon, linkding: LinkdingIconSvelte, aws: AwsIcon, + redshift: AwsIcon, microsoft: MicrosoftIcon, bcrypt: BcryptIcon, google: GoogleIcon, diff --git a/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte b/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte index d472ce6d08..04c4bf7cc0 100644 --- a/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte @@ -7,6 +7,24 @@ /** The name a project reaches when its descriptor names none. */ export const DEFAULT_WAREHOUSE = 'main' + /** + * The resource types a warehouse may point at: `dbt_profile`, which carries a + * `profiles.yml` block verbatim and so reaches any adapter, plus the connection types + * `render_profile` (backend/windmill-worker/src/dbt_profiles.rs) translates. Anything + * else has no way to become a dbt target at all. + */ + export const WAREHOUSE_RESOURCE_TYPES = [ + 'dbt_profile', + 'postgresql', + 'redshift', + 'mysql', + 'snowflake', + 'snowflake_oauth', + 'bigquery', + 'gcp_service_account', + 'databricks' + ].join(',') + export function convertDbtSettingsFromBackend( settings: GetSettingsResponse['dbt_warehouses'] ): DbtSettingsType { @@ -94,10 +112,12 @@ {DEFAULT_WAREHOUSE} when it names none, so a project carries no connection of its own. The name is also what its tables are keyed on in the asset graph (dbt://{DEFAULT_WAREHOUSE}/schema/table), so two projects on one warehouse share their nodes. Each entry points at a resource, and - configuring one here is what makes it available: anyone who may run a dbt script builds with it - and reads its models, without being granted the resource, the same bargain workspace object - storage makes. + >), so two projects on one warehouse share their nodes. Each entry points either at one of + Windmill's own connection resources, whose fields are translated into a dbt target, or at a + dbt_profile resource, which carries a + profiles.yml target as it is and so reaches any adapter dbt has. Configuring + one here is what makes it available: anyone who may run a dbt script builds with it and reads its models, + without being granted the resource, the same bargain workspace object storage makes. @@ -114,20 +134,25 @@ + bind:value={warehouse.name} + inputProps={{ placeholder: DEFAULT_WAREHOUSE }} + class="min-w-32" + /> - + + bind:value={warehouse.target} + inputProps={{ placeholder: 'default' }} + class="min-w-24" + />