Files
windmill/backend/windmill-worker/src/dbt_profiles.rs
T
Ruben FiszelandClaude Opus 5 9fc50a23fb feat: make snowflake_oauth work as a dbt warehouse on every engine (#11095)
* feat: make snowflake_oauth work as a dbt warehouse on every engine

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

* fix: scope the early token refresh to dbt, never follow jail symlinks

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

* fix: lock early token refreshes per account, keep the profile until the new one renders

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

* fix: hold the refresh lock until the new token is written

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

* fix: poll the refresh lock with a bound instead of pinning a connection

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

* refactor: drop the early OAuth refresh from the dbt warehouse route

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

* fix: keep endpoint keys like token_uri in the profile identity

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

* fix: mask access key ids with the secrets they pair with

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011gD7E1cPVhX1kLsB4fAH3b

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-12 00:00:17 +02:00

1573 lines
63 KiB
Rust

//! Rendering a `profiles.yml` from a Windmill warehouse resource, and the
//! adapter package each warehouse needs.
//!
//! Only the fields dbt actually reads are emitted; anything the resource
//! carries for other runtimes is ignored rather than guessed at. Credentials
//! are written into the job dir, which is torn down with the job.
use serde_json::Value;
use windmill_common::error::{self, Error};
use windmill_parser_yaml::dbt::DbtEngine;
/// Written beside `profiles.yml`, and named absolutely in `sslrootcert`: dbt
/// runs with the project as its working directory and hands the path to the
/// driver unchanged.
pub const ROOT_CERT_FILENAME: &str = "server-ca.pem";
/// What a credential's value becomes in [`RenderedProfile::identity`].
const MASKED_CREDENTIAL: &str = "$CREDENTIAL";
/// Whether a target key holds a credential rather than part of the address. By
/// name, because a `dbt_profile` block's keys are its adapter's own: `password`,
/// dbt-postgres's `pass`, `token`, `private_key_passphrase`, `client_secret`,
/// `aws_secret_access_key` and the `key_id` rotated with it, … An endpoint is
/// address even when its name says otherwise: BigQuery's `token_uri` is where the
/// token comes from.
fn is_credential_key(key: &str) -> bool {
let key = key.to_ascii_lowercase();
if key.ends_with("_uri") || key.ends_with("_url") || key.contains("endpoint") {
return false;
}
key == "pass"
|| [
"password",
"passphrase",
"secret",
"token",
"private_key",
"api_key",
"access_key",
"key_id",
]
.iter()
.any(|c| key.contains(c))
}
/// The per-adapter facts, so each adapter states them together and a new one
/// cannot inherit another's by omission. `PG` is the base every arm spreads
/// from: most adapters differ from Postgres only in their name and package.
struct AdapterSpec {
/// As a user would write it, for error messages.
name: &'static str,
/// dbt's own `type:` key in `profiles.yml`.
dbt_type: &'static str,
/// The dbt-core 1.x pip package; empty when the adapter is Fusion-only.
pip_package: &'static str,
default_port: i64,
database_key: &'static str,
requires_enterprise: bool,
/// Overrides `dbt_type` in the licensing error, where the product's own
/// spelling reads better than dbt's driver name.
display_name: Option<&'static str>,
}
impl AdapterSpec {
const PG: AdapterSpec = AdapterSpec {
name: "postgres",
dbt_type: "postgres",
pip_package: "dbt-postgres",
default_port: 5432,
database_key: "dbname",
requires_enterprise: false,
display_name: None,
};
}
/// The adapters Windmill has facts about (decision 9): a field mapping, a pip package, the
/// license gate. NOT the adapters dbt projects may use — `DbtAdapter` carries those by name.
/// The picker offers what reaches one (`WAREHOUSE_RESOURCE_TYPES` in
/// `frontend/.../workspaceSettings/DbtSettings.svelte`), so a mapping change belongs there.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KnownAdapter {
Postgres,
Redshift,
Mysql,
Duckdb,
Clickhouse,
Snowflake,
Bigquery,
Databricks,
Salesforce,
Mssql,
OracleDB,
}
impl KnownAdapter {
/// dbt's OWN `type:` spelling — a separate vocabulary from the resource types
/// below, deliberately. `fabric` is why: Windmill's `fabric` RESOURCE is a SQL
/// Server one, while dbt's `fabric` ADAPTER is its own `dbt-fabric`. Absent
/// here, it falls through to the open path and gets what it asked for.
pub fn from_dbt_type(t: &str) -> Option<Self> {
match t {
"postgres" | "postgresql" => Some(KnownAdapter::Postgres),
"redshift" => Some(KnownAdapter::Redshift),
"mysql" => Some(KnownAdapter::Mysql),
"duckdb" => Some(KnownAdapter::Duckdb),
"clickhouse" => Some(KnownAdapter::Clickhouse),
"snowflake" => Some(KnownAdapter::Snowflake),
"bigquery" => Some(KnownAdapter::Bigquery),
"databricks" => Some(KnownAdapter::Databricks),
"salesforce" => Some(KnownAdapter::Salesforce),
"sqlserver" | "mssql" => Some(KnownAdapter::Mssql),
"oracle" => Some(KnownAdapter::OracleDB),
_ => None,
}
}
pub fn from_resource_type(rt: &str) -> Option<Self> {
match rt {
"postgresql" | "postgres" => Some(KnownAdapter::Postgres),
"redshift" => Some(KnownAdapter::Redshift),
"mysql" => Some(KnownAdapter::Mysql),
"duckdb" => Some(KnownAdapter::Duckdb),
"clickhouse" => Some(KnownAdapter::Clickhouse),
"snowflake" | "snowflake_oauth" => Some(KnownAdapter::Snowflake),
"bigquery" | "gcp_service_account" => Some(KnownAdapter::Bigquery),
"databricks" => Some(KnownAdapter::Databricks),
"salesforce" => Some(KnownAdapter::Salesforce),
"ms_sql_server" | "mssql" | "sqlserver" | "fabric" => Some(KnownAdapter::Mssql),
"oracledb" | "oracle" => Some(KnownAdapter::OracleDB),
_ => None,
}
}
/// Everything that differs per adapter, in one place.
///
/// Adding a warehouse is one arm here, and the match is exhaustive so the
/// compiler demands it. Spread across a method each, two of them carried a
/// `_ =>` default, so a new adapter silently inherited port 5432 and
/// `dbname` instead of failing to build.
fn spec(&self) -> &'static AdapterSpec {
match self {
KnownAdapter::Postgres => &AdapterSpec::PG,
KnownAdapter::Redshift => &AdapterSpec {
name: "redshift",
dbt_type: "redshift",
pip_package: "dbt-redshift",
default_port: 5439,
..AdapterSpec::PG
},
KnownAdapter::Mysql => &AdapterSpec {
name: "mysql",
dbt_type: "mysql",
pip_package: "dbt-mysql",
default_port: 3306,
database_key: "schema",
..AdapterSpec::PG
},
KnownAdapter::Duckdb => &AdapterSpec {
name: "duckdb",
dbt_type: "duckdb",
pip_package: "dbt-duckdb",
..AdapterSpec::PG
},
KnownAdapter::Clickhouse => &AdapterSpec {
name: "clickhouse",
dbt_type: "clickhouse",
pip_package: "dbt-clickhouse",
..AdapterSpec::PG
},
KnownAdapter::Snowflake => &AdapterSpec {
name: "snowflake",
dbt_type: "snowflake",
pip_package: "dbt-snowflake",
..AdapterSpec::PG
},
KnownAdapter::Bigquery => &AdapterSpec {
name: "bigquery",
dbt_type: "bigquery",
pip_package: "dbt-bigquery",
..AdapterSpec::PG
},
KnownAdapter::Databricks => &AdapterSpec {
name: "databricks",
dbt_type: "databricks",
pip_package: "dbt-databricks",
..AdapterSpec::PG
},
// No dbt-core 1.x package exists for it; Fusion has it built in, and
// `provision_core_1x` refuses it by name rather than asking uv to
// install `""`. Pinned by
// `every_adapter_either_names_a_package_or_is_fusion_only`.
KnownAdapter::Salesforce => &AdapterSpec {
name: "salesforce",
dbt_type: "salesforce",
pip_package: "",
..AdapterSpec::PG
},
KnownAdapter::Mssql => &AdapterSpec {
name: "mssql",
dbt_type: "sqlserver",
pip_package: "dbt-sqlserver",
requires_enterprise: true,
display_name: Some("Microsoft SQL server"),
..AdapterSpec::PG
},
KnownAdapter::OracleDB => &AdapterSpec {
name: "oracle",
dbt_type: "oracle",
pip_package: "dbt-oracle",
requires_enterprise: true,
display_name: Some("Oracle DB"),
..AdapterSpec::PG
},
}
}
/// Every adapter, for the tests that must cover all of them. The one list
/// there is: a second would be the thing that goes stale. Test-only, so a
/// release build does not carry a table nothing reads.
#[cfg(test)]
pub const ALL: &'static [KnownAdapter] = &[
KnownAdapter::Postgres,
KnownAdapter::Redshift,
KnownAdapter::Mysql,
KnownAdapter::Duckdb,
KnownAdapter::Clickhouse,
KnownAdapter::Snowflake,
KnownAdapter::Bigquery,
KnownAdapter::Databricks,
KnownAdapter::Salesforce,
KnownAdapter::Mssql,
KnownAdapter::OracleDB,
];
/// Which dbt driver a resource needs, from the fields it carries — a
/// fallback for when the descriptor omits `profile.type`. It picks the
/// adapter only; asset identity is always the workspace warehouse's name,
/// never anything read here.
///
/// Deliberately conservative about the host/port shapes. Windmill's
/// `ms_sql_server` and `oracledb` resources carry the same `host` +
/// `dbname`/`database` fields Postgres does, so a shape check cannot tell
/// them apart — and guessing Postgres for a SQL Server resource would
/// connect dbt-postgres to port 1433 instead of producing the licensing
/// error this design exists to give. So a bare host/database resource is
/// only Postgres when it carries something Postgres-specific, and is
/// otherwise `None`, which asks the user for `profile.type`.
pub fn infer_from_resource(v: &Value) -> Option<Self> {
let has = |k: &str| v.get(k).is_some_and(|x| !x.is_null());
if has("account_identifier") || has("warehouse") {
Some(KnownAdapter::Snowflake)
} else if has("http_path") {
Some(KnownAdapter::Databricks)
} else if has("project_id") && has("client_email") {
Some(KnownAdapter::Bigquery)
} else if has("dbname") && has("host") && (has("sslmode") || has("root_certificate_pem")) {
Some(KnownAdapter::Postgres)
} else {
None
}
}
/// dbt's own `type:` key in `profiles.yml`.
pub fn dbt_type(&self) -> &'static str {
self.spec().dbt_type
}
/// The adapter's name as a user would write it, for error messages.
pub fn name(&self) -> &'static str {
self.spec().name
}
/// The pip package providing this adapter for the dbt-core 1.x engine,
/// whose venv is provisioned on first use. The Rust engines ship their
/// adapters in the binary and never consult this. Empty means no such
/// package exists.
pub fn pip_package(&self) -> &'static str {
self.spec().pip_package
}
/// Only meaningful for the host/port adapters rendered from a resource.
fn default_port(&self) -> i64 {
self.spec().default_port
}
/// dbt spells the database differently per adapter.
fn database_key(&self) -> &'static str {
self.spec().database_key
}
/// The keys a target block spells its database and schema with, mirroring
/// what `render_profile` emits below.
///
/// Read back when the PROJECT owns its `profiles.yml`. Windmill did not
/// write that file, but it still needs the target's database to spell
/// `dbt://` paths the way a rendered profile does: without it every relation
/// qualifies as `<db>.<schema>` while a workspace-warehouse project spells
/// the same table plainly, and the two never share a node.
pub fn target_identity_keys(&self) -> (&'static str, &'static str) {
match self {
KnownAdapter::Bigquery => ("project", "dataset"),
KnownAdapter::Databricks => ("catalog", "schema"),
// `database_key` is what Windmill's own resource spells it, and only
// the adapters `render_profile` translates have one. The rest reach
// dbt through a block they wrote themselves, which spells it the way
// dbt does.
KnownAdapter::Postgres | KnownAdapter::Redshift | KnownAdapter::Mysql => {
(self.database_key(), "schema")
}
_ => ("database", "schema"),
}
}
/// Whether this adapter needs an enterprise license.
///
/// The boundary is not invented for dbt: it mirrors the two native script
/// languages that are still enterprise-gated. Every other warehouse dbt can
/// reach has a CE `ScriptLang`, so gating its dbt adapter would be stricter
/// than running the same query natively.
pub fn requires_enterprise(&self) -> bool {
self.spec().requires_enterprise
}
/// The name used in the licensing error, matching how the native languages
/// name themselves.
fn display_name(&self) -> &'static str {
let spec = self.spec();
spec.display_name.unwrap_or(spec.dbt_type)
}
}
/// The adapter a profile connects with: dbt's own `type:`, plus whatever Windmill knows.
/// Open by construction — a `dbt_profile` carries a block Windmill never has to understand,
/// so an unknown adapter is still rendered, licensed and identified. INSTALLING one is a
/// separate question, gated by `ensure_adapter_installable`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DbtAdapter {
known: Option<KnownAdapter>,
/// dbt's own `type:` spelling, lowercased. Also the pip package's suffix.
name: String,
}
impl DbtAdapter {
/// An adapter as dbt spells it, known or not. The name reaches a pip requirement
/// and a venv path, so it is confined to what an adapter name can be rather than
/// escaped at each use: a leading `-` is a pip flag, a `/` or `..` a path segment.
pub fn from_dbt_type(t: &str) -> error::Result<Self> {
let name = t.trim().to_ascii_lowercase();
let shaped = name.len() <= 40
&& name.starts_with(|c: char| c.is_ascii_alphanumeric())
&& name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-');
if !shaped {
return Err(Error::BadRequest(format!(
"`{t}` is not a dbt adapter name: dbt spells one with letters, digits, `_` and \
`-`, as in `postgres` or `trino`"
)));
}
// Normalised to the adapter's own dbt spelling when one resolves, so two
// names for one adapter are one value: `PartialEq` covers `name` too, and
// `profile.type: mssql` over a `sqlserver` target would otherwise be
// rejected by a message naming the same adapter on both sides.
let known = KnownAdapter::from_dbt_type(&name);
let name = known.map_or(name, |k| k.dbt_type().to_string());
Ok(Self { known, name })
}
/// The adapter a `dbt_profile` states, from the `type` of the block its value is.
/// Called only for that resource TYPE, never sniffed: Windmill's bigquery resource
/// is a service-account JSON and says `type: service_account`.
pub fn stated_by_dbt_profile(v: &Value) -> error::Result<Self> {
let stated = v.get("type").and_then(|t| t.as_str()).ok_or_else(|| {
Error::BadRequest(
"a `dbt_profile` resource names its adapter in `type`, as a `profiles.yml` output \
does; this one has none"
.to_string(),
)
})?;
Self::from_dbt_type(stated)
}
/// The adapter's facts, when it is one Windmill has any.
pub fn known(&self) -> Option<KnownAdapter> {
self.known
}
/// dbt's own `type:` key in `profiles.yml`.
pub fn dbt_type(&self) -> &str {
self.known.map_or(self.name.as_str(), |k| k.dbt_type())
}
/// The adapter's name as a user would write it, for error messages.
pub fn name(&self) -> &str {
self.known.map_or(self.name.as_str(), |k| k.name())
}
/// The pip package providing this adapter for the dbt-core 1.x engine.
/// Empty means no such package exists and only Fusion has it.
pub fn pip_package(&self) -> String {
match self.known {
Some(k) => k.pip_package().to_string(),
None => format!("dbt-{}", self.name),
}
}
/// The keys a target block spells its database and schema with.
/// An unknown adapter is read with dbt's ordinary pair.
pub fn target_identity_keys(&self) -> (&'static str, &'static str) {
self.known
.map_or(("database", "schema"), |k| k.target_identity_keys())
}
/// Whether this adapter needs an enterprise license. Never for an unknown
/// one: the gate mirrors the two native warehouse languages, and an adapter
/// with no Windmill runtime behind it is not one of them.
pub fn requires_enterprise(&self) -> bool {
self.known.is_some_and(|k| k.requires_enterprise())
}
fn display_name(&self) -> &str {
self.known.map_or(self.name.as_str(), |k| k.display_name())
}
}
impl From<KnownAdapter> for DbtAdapter {
fn from(known: KnownAdapter) -> Self {
Self { known: Some(known), name: known.dbt_type().to_string() }
}
}
/// Whether this build may use the enterprise-only adapters.
///
/// Two conditions, both required. `LICENSE_KEY_VALID` alone is not enough: the
/// OSS variant of `ee_oss` initializes it to `true`, so reading it on a CE
/// build would wave everything through. The `cfg` establishes it is an
/// enterprise build; the atomic then reports whether that build's key actually
/// verified.
fn enterprise_licensed() -> bool {
#[cfg(feature = "enterprise")]
{
windmill_common::ee_oss::LICENSE_KEY_VALID.load(std::sync::atomic::Ordering::Relaxed)
}
#[cfg(not(feature = "enterprise"))]
{
false
}
}
/// Reject an enterprise-only adapter on an unlicensed build.
///
/// Unlike the native languages, this cannot be a compile-time gate: there is
/// one dbt executor and the adapter is only known once the profile resolves. So
/// it is checked at both deploy and run — and it says so plainly, rather than
/// letting the run fail later with a connection error the user cannot act on.
pub fn ensure_adapter_licensed(adapter: &DbtAdapter) -> error::Result<()> {
if adapter.requires_enterprise() && !enterprise_licensed() {
return Err(Error::BadRequest(format!(
"{} is only available with an enterprise license",
adapter.display_name()
)));
}
Ok(())
}
fn s(v: &Value, k: &str) -> Option<String> {
v.get(k)
.and_then(|x| x.as_str())
.map(|x| x.to_string())
.filter(|x| !x.is_empty())
}
/// A resource's port, which may arrive as a JSON number or as a string —
/// resources are user-supplied JSON and nothing coerces the field. Reading only
/// `as_i64` sent `"5433"` to the adapter default, which connects to whatever
/// listens there and reports nothing; a value that is present but not a port is
/// refused rather than replaced.
fn port_of(resource: &Value, default: i64) -> error::Result<i64> {
let bad = |v: &Value| {
Error::BadRequest(format!(
"resource `port` must be a number, got `{v}`; correct the resource"
))
};
match resource.get("port") {
None | Some(Value::Null) => Ok(default),
Some(Value::Number(n)) => n.as_i64().ok_or_else(|| bad(&Value::Number(n.clone()))),
Some(Value::String(s)) if s.trim().is_empty() => Ok(default),
Some(Value::String(s)) => s
.trim()
.parse::<i64>()
.map_err(|_| bad(&Value::String(s.clone()))),
Some(other) => Err(bad(other)),
}
}
/// The rendered `profiles.yml` body plus the `(schema, database)` the target
/// resolves to. The caller needs those two to spell the `dbt://` asset paths
/// of models that do not override them.
#[derive(Debug)]
pub struct RenderedProfile {
pub yaml: String,
/// `yaml` with every credential's value masked: what run identity hashes. A
/// rotated token or password is still the same connection, and an OAuth token
/// rotates every few minutes, so hashing it would refuse nearly every retry.
pub identity: String,
pub schema: Option<String>,
pub database: Option<String>,
/// A private CA the caller must write next to `profiles.yml`, under the
/// file name `sslrootcert` already points at. Returned rather than written
/// here so this stays a pure renderer.
pub root_certificate_pem: Option<String>,
}
/// Render a single-target `profiles.yml` for `profile_name`/`target`.
///
/// `threads` is the descriptor's, defaulted by dbt when absent. `schema` comes
/// from the descriptor when set, else from the resource, else from dbt's own
/// per-adapter default — dbt errors out clearly when it ends up missing, which
/// is a better failure than a Windmill-invented default.
#[allow(clippy::too_many_arguments)]
pub fn render_profile(
adapter: &DbtAdapter,
// The engines read some keys differently; see the Snowflake token below.
engine: DbtEngine,
resource: &Value,
profile_name: &str,
target: &str,
threads: Option<u32>,
schema_override: Option<&str>,
// Where `profiles.yml` (and any root certificate) is written. dbt runs with
// the PROJECT as its working directory and hands `sslrootcert` to the
// driver unchanged, so a path relative to the profiles dir would be
// resolved against the project and never found.
profiles_dir: &std::path::Path,
) -> error::Result<RenderedProfile> {
// Past here the resource is one of Windmill's own connection types, which
// becomes a target only through a field mapping below — so an adapter
// Windmill has no facts about cannot be rendered from one at all.
let adapter = adapter.known().ok_or_else(|| {
Error::BadRequest(format!(
"no Windmill resource translates into a `{}` target; point the warehouse at a \
`dbt_profile` resource, whose value is the profiles.yml block itself",
adapter.dbt_type()
))
})?;
let mut out: Vec<(String, ProfileValue)> = vec![("type".into(), quoted(adapter.dbt_type()))];
let mut schema = schema_override.map(|x| x.to_string());
let database;
match adapter {
// Redshift and MySQL take the same host/port/user/password/database
// shape as Postgres in both dbt and Windmill's resource types, so one
// arm renders all three; only the default port and the database key
// differ.
KnownAdapter::Postgres | KnownAdapter::Redshift | KnownAdapter::Mysql => {
let host = s(resource, "host").ok_or_else(|| {
Error::BadRequest(format!("{} resource has no `host`", adapter.dbt_type()))
})?;
let dbname = s(resource, "dbname")
.or_else(|| s(resource, "database"))
.or_else(|| s(resource, "service_name"))
.ok_or_else(|| {
Error::BadRequest(format!(
"{} resource has no `dbname`/`database`",
adapter.dbt_type()
))
})?;
out.push(("host".into(), quoted(&host)));
out.push((
"port".into(),
// dbt validates `port` against a JSON schema that demands an
// integer; a quoted scalar is rejected outright.
ProfileValue::Number(port_of(resource, adapter.default_port())?),
));
out.push((adapter.database_key().into(), quoted(&dbname)));
database = Some(dbname.clone());
if let Some(u) = s(resource, "user") {
out.push(("user".into(), quoted(&u)));
}
if let Some(p) = s(resource, "password") {
out.push(("password".into(), quoted(&p)));
}
if let Some(m) = s(resource, "sslmode") {
out.push(("sslmode".into(), quoted(&m)));
}
// Under `verify-ca`/`verify-full` a resource's private CA is the
// only way the connection can succeed; without it libpq looks for a
// default certificate that is not there, and the same field is what
// identifies the resource as Postgres in the first place.
if s(resource, "root_certificate_pem").is_some() {
out.push((
"sslrootcert".into(),
quoted(&profiles_dir.join(ROOT_CERT_FILENAME).to_string_lossy()),
));
}
schema = match adapter {
// Already emitted as the database key; reported back so the
// caller can spell `dbt://` paths with it.
KnownAdapter::Mysql => Some(dbname.clone()),
_ => schema
.or_else(|| s(resource, "schema"))
.or(Some("public".into())),
};
}
// Their Windmill resources do not carry what dbt needs — an `oracledb`
// resource has no host/service, dbt-sqlserver needs an ODBC `driver` the
// images lack — so a rendered profile could not connect. They are reached
// through a target written for them instead.
KnownAdapter::Duckdb
| KnownAdapter::Clickhouse
| KnownAdapter::Salesforce
| KnownAdapter::Mssql
| KnownAdapter::OracleDB => {
return Err(Error::BadRequest(format!(
"a `{}` resource carries nothing dbt can connect with; point the warehouse at a \
`dbt_profile` resource, whose value is the profiles.yml block itself, or \
`profile.profiles_yml` at the project's own profiles.yml",
adapter.dbt_type()
)));
}
KnownAdapter::Snowflake => {
let account = s(resource, "account_identifier")
.or_else(|| s(resource, "account"))
.ok_or_else(|| {
Error::BadRequest("snowflake resource has no `account_identifier`".to_string())
})?;
out.push(("account".into(), quoted(&account)));
if let Some(u) = s(resource, "username").or_else(|| s(resource, "user")) {
out.push(("user".into(), quoted(&u)));
}
// Key-pair is Windmill's own snowflake resource shape; the
// `snowflake_oauth` type carries an access token instead, which needs
// an `authenticator` saying so. dbt-core 1.x reads `oauth` + `token`
// as one. The Rust engines read `oauth` as the refresh-token flow and
// refuse a profile without client credentials, and send the same
// login for `jwt`, which dbt-snowflake has only from 1.9, while the
// 1.x engine still resolves 1.8.
if let Some(t) = s(resource, "token").or_else(|| s(resource, "access_token")) {
let authenticator = match engine {
DbtEngine::DbtCore1x => "oauth",
DbtEngine::DbtCore2x | DbtEngine::Fusion => "jwt",
};
out.push(("authenticator".into(), quoted(authenticator)));
out.push(("token".into(), quoted(&t)));
} else if let Some(k) = s(resource, "private_key") {
out.push(("private_key".into(), quoted(&k)));
if let Some(pp) = s(resource, "private_key_passphrase") {
out.push(("private_key_passphrase".into(), quoted(&pp)));
}
} else if let Some(p) = s(resource, "password") {
out.push(("password".into(), quoted(&p)));
}
for k in ["warehouse", "role"] {
if let Some(v) = s(resource, k) {
out.push((k.into(), quoted(&v)));
}
}
// dbt-snowflake requires one, and a resource from the OAuth connect flow
// starts without it: naming the field beats dbt's schema error.
let db = s(resource, "database").ok_or_else(|| {
Error::BadRequest(
"a Snowflake target needs a database; add `database` to the warehouse's \
resource"
.to_string(),
)
})?;
out.push(("database".into(), quoted(&db)));
database = Some(db);
schema = schema.or_else(|| s(resource, "schema"));
}
KnownAdapter::Bigquery => {
// Windmill's bigquery resource is the raw service-account JSON, so
// hand dbt the same document via `method: service-account-json`
// rather than re-deriving individual fields.
out.push(("method".into(), quoted("service-account-json")));
let project = s(resource, "project_id").ok_or_else(|| {
Error::BadRequest("bigquery resource has no `project_id`".to_string())
})?;
out.push(("project".into(), quoted(&project)));
database = Some(project);
// A service-account JSON has no dataset, so unless the resource was
// extended with one the descriptor must supply it — dbt rejects a
// BigQuery target without `dataset`, and failing here names the
// missing field instead.
schema = schema
.or_else(|| s(resource, "dataset"))
.or_else(|| s(resource, "schema"));
if schema.is_none() {
return Err(Error::BadRequest(
"a BigQuery target needs a dataset; set `profile.schema` in the descriptor"
.to_string(),
));
}
}
KnownAdapter::Databricks => {
// Windmill's databricks resource spells the workspace `workspace_url`, and
// spells it as a full URL; dbt wants a bare hostname under `host`.
let host = s(resource, "host")
.or_else(|| s(resource, "workspace_url").map(|u| bare_host(&u)))
.ok_or_else(|| {
Error::BadRequest(
"databricks resource has no `workspace_url`/`host`".to_string(),
)
})?;
out.push(("host".into(), quoted(&host)));
for (k, rk) in [("http_path", "http_path"), ("token", "token")] {
let v = s(resource, rk).ok_or_else(|| {
Error::BadRequest(format!("databricks resource has no `{rk}`"))
})?;
out.push((k.into(), quoted(&v)));
}
database = s(resource, "catalog");
if let Some(c) = database.clone() {
out.push(("catalog".into(), quoted(&c)));
}
schema = schema.or_else(|| s(resource, "schema"));
}
}
// dbt-mysql has no database/schema distinction: its `schema` key IS the
// database, already emitted above. Pushing the generic one too would put
// two `schema` keys in one target — an invalid profile, or one silently
// pointing at the wrong database.
if adapter != KnownAdapter::Mysql {
if let Some(sc) = schema.clone() {
// dbt-bigquery spells it `dataset`; every other adapter says
// `schema`. Emitting `schema` there produces a profile dbt rejects.
let key = match adapter {
KnownAdapter::Bigquery => "dataset",
_ => "schema",
};
out.push((key.into(), quoted(&sc)));
}
}
if let Some(t) = threads {
out.push(("threads".into(), ProfileValue::Number(t as i64)));
}
// Quoted, keys included. `profile_name` comes from the project's own
// `dbt_project.yml` and `target` from the descriptor, so both are the
// author's text: a name like `prod # x` silently truncates the mapping, and
// a newline in one opens a sibling key of the caller's choosing.
let (qp, qt) = (yaml_scalar(profile_name), yaml_scalar(target));
let mut yaml = format!("{qp}:\n target: {qt}\n outputs:\n {qt}:\n");
let mut identity = yaml.clone();
for (k, v) in &out {
yaml.push_str(&format!(" {k}: {}\n", v.render()));
let shown = if is_credential_key(k) {
yaml_scalar(MASKED_CREDENTIAL)
} else {
v.render()
};
identity.push_str(&format!(" {k}: {shown}\n"));
}
// The service-account document is a nested mapping, not a scalar.
if adapter == KnownAdapter::Bigquery {
yaml.push_str(" keyfile_json:\n");
identity.push_str(" keyfile_json:\n");
let obj = resource
.as_object()
.ok_or_else(|| Error::BadRequest("bigquery resource is not an object".to_string()))?;
for (k, v) in obj {
if let Some(v) = v.as_str() {
yaml.push_str(&format!(" {}: {}\n", yaml_scalar(k), yaml_scalar(v)));
let shown = if is_credential_key(k) {
MASKED_CREDENTIAL
} else {
v
};
identity.push_str(&format!(
" {}: {}\n",
yaml_scalar(k),
yaml_scalar(shown)
));
}
}
}
Ok(RenderedProfile {
yaml,
identity,
schema,
database,
root_certificate_pem: matches!(adapter, KnownAdapter::Postgres)
.then(|| s(resource, "root_certificate_pem"))
.flatten(),
})
}
/// Render a `dbt_profile`: its value IS one entry of `profiles.yml`'s `outputs` map, so it
/// is emitted as it stands — nothing lifted out or renamed, which is the point of the type.
/// Only what dbt cannot take literally is handled: the adapter's `type`, a certificate that
/// is a PEM body rather than a path, and the two keys a descriptor may override.
pub fn render_dbt_profile(
adapter: &DbtAdapter,
block: &serde_json::Map<String, Value>,
profile_name: &str,
target: &str,
threads: Option<u32>,
schema_override: Option<&str>,
profiles_dir: &std::path::Path,
) -> error::Result<RenderedProfile> {
let (database_key, schema_key) = adapter.target_identity_keys();
let root_certificate_pem = block
.get("root_certificate_pem")
.and_then(|v| v.as_str())
.filter(|v| !v.is_empty())
.map(|v| v.to_string());
let (qp, qt) = (yaml_scalar(profile_name), yaml_scalar(target));
let mut yaml = format!("{qp}:\n target: {qt}\n outputs:\n {qt}:\n");
// Keys quoted throughout this block, the block's own and Windmill's alike: a
// target whose `"host"` is quoted and whose `schema` is not reads as a bug.
yaml.push_str(&format!(
" \"type\": {}\n",
yaml_scalar(adapter.dbt_type())
));
let mut identity = yaml.clone();
for (k, v) in block {
// A null is an optional field the resource form left unset, and dbt
// validates several keys against a schema that rejects one.
if k == "type" || k == "root_certificate_pem" || v.is_null() {
continue;
}
// Only when Windmill writes one of its own, which would otherwise be a second
// `sslrootcert`. A path with no PEM beside it is the block's own trust source —
// a CA baked into the image or mounted on the worker — and dropping it changes
// what the connection verifies against.
if k == "sslrootcert" && root_certificate_pem.is_some() {
continue;
}
if (k == schema_key && schema_override.is_some()) || (k == "threads" && threads.is_some()) {
continue;
}
emit_entry(&mut yaml, 6, k, v, false);
emit_entry(&mut identity, 6, k, v, true);
}
let mut tail = String::new();
if root_certificate_pem.is_some() {
tail.push_str(&format!(
" \"sslrootcert\": {}\n",
yaml_scalar(&profiles_dir.join(ROOT_CERT_FILENAME).to_string_lossy())
));
}
if let Some(sc) = schema_override {
tail.push_str(&format!(
" {}: {}\n",
yaml_scalar(schema_key),
yaml_scalar(sc)
));
}
if let Some(t) = threads {
tail.push_str(&format!(" \"threads\": {t}\n"));
}
yaml.push_str(&tail);
identity.push_str(&tail);
let str_key = |k: &str| block.get(k).and_then(|v| v.as_str()).map(|v| v.to_string());
Ok(RenderedProfile {
yaml,
identity,
schema: schema_override
.map(|x| x.to_string())
.or_else(|| str_key(schema_key)),
database: str_key(database_key),
root_certificate_pem,
})
}
/// Emit one target key, nesting as deep as the value goes — an adapter's credential can be
/// a mapping (bigquery's `keyfile_json`) or a list. Keys are quoted like values: one nothing
/// here enumerates is as free-form as a password. `mask` writes credentials as
/// [`MASKED_CREDENTIAL`], at any depth, for [`RenderedProfile::identity`].
fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value, mask: bool) {
out.push_str(&format!("{}{}:", " ".repeat(indent), yaml_scalar(key)));
emit_value(out, indent, v, mask, mask && is_credential_key(key));
}
/// The value half, after `key:`. An empty collection is emitted INLINE: a block with no
/// children reads back as `null`, so `extensions: []` would reach the adapter as a missing
/// value rather than the empty list dbt was handed.
///
/// `credential` masks scalars only. A collection under a credential-named key is still
/// walked: dbt-duckdb's `secrets:` list holds the endpoint and scope that say which
/// connection it is, each entry judged by its own key.
fn emit_value(out: &mut String, indent: usize, v: &Value, mask: bool, credential: bool) {
match v {
Value::Object(m) => {
// A null is an optional field the resource form left unset, and dbt validates
// several keys against a schema that rejects one.
let kept: Vec<_> = m.iter().filter(|(_, v)| !v.is_null()).collect();
if kept.is_empty() {
out.push_str(" {}\n");
return;
}
out.push('\n');
for (k, v) in kept {
emit_entry(out, indent + 2, k, v, mask);
}
}
Value::Array(items) => {
if items.is_empty() {
out.push_str(" []\n");
return;
}
out.push('\n');
let pad = " ".repeat(indent + 2);
for item in items {
out.push_str(&pad);
out.push('-');
emit_value(out, indent + 2, item, mask, credential);
}
}
_ if credential => out.push_str(&format!(" {}\n", yaml_scalar(MASKED_CREDENTIAL))),
_ => out.push_str(&format!(" {}\n", yaml_value(v))),
}
}
/// A scalar as dbt reads it: quoted when it is text, bare when it is not.
/// `port`, `threads` and the boolean toggles adapters carry are validated
/// against a JSON schema that rejects the quoted form.
fn yaml_value(v: &Value) -> String {
match v {
Value::String(s) => yaml_scalar(s),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
other => yaml_scalar(&other.to_string()),
}
}
/// One rendered `profiles.yml` value. Strings are always quoted so credential
/// content cannot inject profile keys; numbers must NOT be, because dbt
/// validates `port` and `threads` against a JSON schema that demands integers
/// and rejects the quoted form outright.
enum ProfileValue {
Str(String),
Number(i64),
}
impl ProfileValue {
fn render(&self) -> String {
match self {
ProfileValue::Str(s) => yaml_scalar(s),
ProfileValue::Number(n) => n.to_string(),
}
}
}
fn quoted(v: &str) -> ProfileValue {
ProfileValue::Str(v.to_string())
}
/// A workspace URL as dbt wants it: the hostname alone. Windmill's databricks
/// resource carries the full deployment URL, while dbt-databricks builds its own
/// URL from `host`, so a scheme left in place produces `https://https://…`.
fn bare_host(url: &str) -> String {
url.trim()
.trim_start_matches("https://")
.trim_start_matches("http://")
.trim_end_matches('/')
.to_string()
}
/// Emit a YAML scalar that survives any credential content. Always
/// double-quoted with the two characters that can terminate or escape inside a
/// double-quoted scalar escaped, so a password containing `#`, `:`, a newline
/// or a quote cannot break out of its value and inject profile keys.
fn yaml_scalar(v: &str) -> String {
let escaped = v
.replace('\\', "\\\\")
.replace('"', "\\\"")
.replace('\n', "\\n")
.replace('\r', "\\r");
format!("\"{escaped}\"")
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn renders_postgres_target() {
let r = json!({"host": "db.internal", "port": 5433, "user": "u", "password": "p",
"dbname": "warehouse", "sslmode": "require"});
let p = render_profile(
&KnownAdapter::Postgres.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"prod",
Some(8),
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
// Quoted: the profile name and target are the author's text, so they are
// rendered as scalars rather than as bare keys.
assert!(
p.yaml.contains("\"wm\":\n target: \"prod\"\n"),
"{}",
p.yaml
);
assert!(p.yaml.contains(" type: \"postgres\"\n"));
// dbt's profile schema types these as integers, so they must not be
// quoted like the credential scalars around them.
assert!(p.yaml.contains(" port: 5433\n"));
assert!(p.yaml.contains(" threads: 8\n"));
assert_eq!(p.database.as_deref(), Some("warehouse"));
assert_eq!(p.schema.as_deref(), Some("public"));
}
#[test]
fn redshift_falls_back_to_its_own_port() {
// Redshift is rendered from the same host/port shape as Postgres but
// does not answer on Postgres's port.
let r = json!({"host": "cluster.redshift.amazonaws.com", "user": "u",
"password": "p", "dbname": "warehouse"});
let p = render_profile(
&KnownAdapter::Redshift.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert!(p.yaml.contains(" port: 5439\n"), "{}", p.yaml);
}
// The point of `dbt_profile`: an adapter with no field mapping still
// connects, because its value is a block dbt wrote and reaches dbt as it is.
#[test]
fn dbt_profile_renders_its_block_verbatim() {
let r = json!({"type": "clickhouse", "host": "ch.internal", "port": 8123,
"secure": true, "user": "u", "password": "p", "schema": "analytics"});
let p = render_dbt_profile(
&KnownAdapter::Clickhouse.into(),
r.as_object().unwrap(),
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert!(
p.yaml.contains(" \"type\": \"clickhouse\"\n"),
"{}",
p.yaml
);
assert!(
p.yaml.contains(" \"host\": \"ch.internal\"\n"),
"{}",
p.yaml
);
// dbt types these in its own schema and rejects the quoted form.
assert!(p.yaml.contains(" \"port\": 8123\n"), "{}", p.yaml);
assert!(p.yaml.contains(" \"secure\": true\n"), "{}", p.yaml);
// The identity a `dbt://` path is spelled with, read back from the block.
assert_eq!(p.schema.as_deref(), Some("analytics"));
}
// Two `schema` keys in one target is a profile dbt rejects, or one silently
// pointing at the schema the descriptor meant to override.
#[test]
fn dbt_profile_schema_override_replaces_the_block_key() {
let r = json!({"type": "clickhouse", "host": "h", "schema": "raw"});
let p = render_dbt_profile(
&KnownAdapter::Clickhouse.into(),
r.as_object().unwrap(),
"wm",
"prod",
None,
Some("staging"),
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert_eq!(p.yaml.matches("\"schema\":").count(), 1, "{}", p.yaml);
assert!(p.yaml.contains("\"schema\": \"staging\"\n"), "{}", p.yaml);
assert_eq!(p.schema.as_deref(), Some("staging"));
}
// `fabric` is a distinct dbt adapter AND a Windmill resource type for SQL
// Server. Resolving dbt's `type:` through the resource table installed
// dbt-sqlserver for it, under an enterprise gate, and never said Fabric.
#[test]
fn a_dbt_type_is_not_a_resource_type() {
let fabric = DbtAdapter::from_dbt_type("fabric").unwrap();
assert_eq!(fabric.dbt_type(), "fabric");
assert_eq!(fabric.pip_package(), "dbt-fabric");
assert!(!fabric.requires_enterprise());
// The Windmill resource type keeps mapping where it always did.
assert_eq!(
KnownAdapter::from_resource_type("fabric"),
Some(KnownAdapter::Mssql)
);
}
// `PartialEq` covers the carried name, so two spellings of one adapter must
// normalise or the descriptor/resource agreement check rejects a valid
// config with a message naming the same adapter on both sides.
#[test]
fn two_spellings_of_one_adapter_are_one_value() {
for (a, b) in [("postgres", "postgresql"), ("sqlserver", "mssql")] {
assert_eq!(
DbtAdapter::from_dbt_type(a).unwrap(),
DbtAdapter::from_dbt_type(b).unwrap(),
"{a} vs {b}"
);
}
}
// An adapter Windmill has no facts about is still an adapter: this is what
// "whatever dbt supports" rests on.
#[test]
fn an_unknown_adapter_is_carried_by_name() {
let stated = DbtAdapter::stated_by_dbt_profile(&json!({"type": "trino", "host": "h"}))
.unwrap();
assert_eq!(stated.dbt_type(), "trino");
assert_eq!(stated.pip_package(), "dbt-trino");
assert!(stated.known().is_none());
// A block with no adapter cannot be rendered, and says which key is missing.
assert!(DbtAdapter::stated_by_dbt_profile(&json!({"host": "h"})).is_err());
}
// A block whose CA is a path on the worker keeps it: Windmill only takes the
// key over when it has a PEM of its own to point at.
#[test]
fn a_path_only_sslrootcert_survives() {
let r = json!({"type": "postgres", "host": "h", "sslmode": "verify-full",
"sslrootcert": "/etc/ssl/certs/warehouse-ca.pem"});
let p = render_dbt_profile(
&KnownAdapter::Postgres.into(),
r.as_object().unwrap(),
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert!(
p.yaml
.contains("\"sslrootcert\": \"/etc/ssl/certs/warehouse-ca.pem\"\n"),
"{}",
p.yaml
);
// And a PEM in the block still wins, exactly once.
let r = json!({"type": "postgres", "host": "h", "sslrootcert": "/ignored",
"root_certificate_pem": "-----BEGIN CERTIFICATE-----\nx\n"});
let p = render_dbt_profile(
&KnownAdapter::Postgres.into(),
r.as_object().unwrap(),
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert_eq!(p.yaml.matches("sslrootcert").count(), 1, "{}", p.yaml);
assert!(p.yaml.contains(ROOT_CERT_FILENAME), "{}", p.yaml);
}
// dbt hands these to the adapter as it finds them, so a collection must survive the
// round trip: a block with no children reads back as `null`, not as `[]` or `{}`.
#[test]
fn collections_keep_their_type() {
let r = json!({"type": "duckdb", "extensions": [], "settings": {},
"attach": [{"path": "raw.db", "read_only": true}],
"matrix": [["a", 1], []], "plugins": ["excel", "json"]});
let p = render_dbt_profile(
&DbtAdapter::from_dbt_type("duckdb").unwrap(),
r.as_object().unwrap(),
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
// Parsed back rather than string-matched: the point is what a YAML reader sees.
let y: serde_json::Value = serde_yml::from_str(&p.yaml).expect(&p.yaml);
let t = &y["wm"]["outputs"]["prod"];
assert_eq!(t["extensions"], json!([]), "{}", p.yaml);
assert_eq!(t["settings"], json!({}), "{}", p.yaml);
assert_eq!(t["attach"], json!([{"path": "raw.db", "read_only": true}]), "{}", p.yaml);
assert_eq!(t["matrix"], json!([["a", 1], []]), "{}", p.yaml);
assert_eq!(t["plugins"], json!(["excel", "json"]), "{}", p.yaml);
}
// An unknown adapter's name becomes `dbt-<name>` in a pip requirement and a
// venv path, both on the host and outside the jail.
#[test]
fn an_adapter_name_cannot_be_a_pip_flag_or_a_path() {
for bad in [
"--index-url=http://evil",
"-e .",
"../../etc/passwd",
"dbt postgres",
"",
] {
assert!(DbtAdapter::from_dbt_type(bad).is_err(), "{bad}");
}
assert_eq!(
DbtAdapter::from_dbt_type("TRINO").unwrap().dbt_type(),
"trino"
);
}
// Windmill's databricks resource spells the workspace as a full URL under
// `workspace_url`; dbt-databricks builds its own URL from a bare `host`.
#[test]
fn databricks_takes_its_host_from_the_workspace_url() {
let r = json!({"workspace_url": "https://dbc-a1b2.cloud.databricks.com/",
"http_path": "/sql/1.0/warehouses/x", "token": "t"});
let p = render_profile(
&KnownAdapter::Databricks.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert!(
p.yaml
.contains(" host: \"dbc-a1b2.cloud.databricks.com\"\n"),
"{}",
p.yaml
);
}
// A resource's private CA is the only way a `verify-full` connection can
// succeed, and `root_certificate_pem` is also what identifies the resource as
// Postgres — forwarding one and dropping the other is incoherent.
#[test]
fn postgres_forwards_its_root_certificate() {
let r = json!({"host": "h", "dbname": "d", "sslmode": "verify-full",
"root_certificate_pem": "-----BEGIN CERTIFICATE-----\nx\n"});
let p = render_profile(
&KnownAdapter::Postgres.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
// Absolute: dbt runs with the PROJECT as its cwd and hands this to the
// driver unchanged, so a profiles-relative path would never be found.
let cert_path = std::path::Path::new("/tmp/p").join(ROOT_CERT_FILENAME);
assert!(
p.yaml.contains(&format!(
" sslrootcert: {}\n",
yaml_scalar(&cert_path.to_string_lossy())
)),
"{}",
p.yaml
);
assert_eq!(
p.root_certificate_pem.as_deref(),
Some("-----BEGIN CERTIFICATE-----\nx\n")
);
// No CA configured, no dangling sslrootcert pointing at a missing file.
let plain = json!({"host": "h", "dbname": "d", "sslmode": "require"});
let p = render_profile(
&KnownAdapter::Postgres.into(),
DbtEngine::DbtCore1x,
&plain,
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert!(!p.yaml.contains("sslrootcert"));
assert_eq!(p.root_certificate_pem, None);
}
fn snowflake(engine: DbtEngine, r: &Value) -> error::Result<RenderedProfile> {
render_profile(
&KnownAdapter::Snowflake.into(),
engine,
r,
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
}
// `snowflake_oauth` carries an access token, which only one `authenticator`
// per engine accepts: the Rust engines refuse `oauth` without client
// credentials, and dbt-snowflake before 1.9 has no `jwt`.
#[test]
fn snowflake_oauth_names_its_token_per_engine() {
let r = json!({"account_identifier": "acc", "token": "tok", "database": "db"});
for (engine, authenticator) in [
(DbtEngine::DbtCore1x, "oauth"),
(DbtEngine::DbtCore2x, "jwt"),
(DbtEngine::Fusion, "jwt"),
] {
let p = snowflake(engine, &r).unwrap();
assert!(
p.yaml
.contains(&format!(" authenticator: \"{authenticator}\"\n")),
"{engine:?}: {}",
p.yaml
);
assert!(p.yaml.contains(" token: \"tok\"\n"));
}
let err = snowflake(
DbtEngine::DbtCore1x,
&json!({"account_identifier": "acc", "token": "tok"}),
)
.unwrap_err()
.to_string();
assert!(err.contains("add `database`"), "{err}");
}
// Run identity has to survive a credential rotating, which an OAuth token does
// every few minutes, and still change when the connection moves.
#[test]
fn identity_masks_credentials_but_not_the_connection() {
let rendered = |account: &str, token: &str| {
snowflake(
DbtEngine::DbtCore1x,
&json!({"account_identifier": account, "token": token, "database": "db"}),
)
.unwrap()
.identity
};
assert_eq!(rendered("acc", "t1"), rendered("acc", "t2"));
assert_ne!(rendered("acc", "t1"), rendered("other", "t1"));
// A `dbt_profile` block, whose keys are the adapter's own, nested ones too.
let block = |adapter: KnownAdapter, v: Value| {
render_dbt_profile(
&adapter.into(),
v.as_object().unwrap(),
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap()
.identity
};
let bq = |project: &str, key: &str, token_uri: &str| {
block(
KnownAdapter::Bigquery,
json!({"type": "bigquery", "project": project, "dataset": "d",
"keyfile_json": {"client_email": "e", "private_key": key,
"token_uri": token_uri}}),
)
};
assert_eq!(bq("p", "k1", "t"), bq("p", "k2", "t"));
assert_ne!(bq("p", "k1", "t"), bq("q", "k1", "t"));
assert_ne!(bq("p", "k1", "t"), bq("p", "k1", "elsewhere"));
// Only scalars are masked: a `secrets:` entry still names its endpoint.
// A rotated access key changes its id with its secret.
let duck = |endpoint: &str, secret: &str| {
block(
KnownAdapter::Duckdb,
json!({"type": "duckdb", "path": "x.duckdb",
"secrets": [{"type": "s3", "endpoint": endpoint,
"key_id": format!("id-{secret}"), "secret": secret}]}),
)
};
assert_eq!(duck("s3.a", "k1"), duck("s3.a", "k2"));
assert_ne!(duck("s3.a", "k1"), duck("s3.b", "k1"));
let pg = |pass: &str| {
block(
KnownAdapter::Postgres,
json!({"type": "postgres", "host": "h", "user": "u", "pass": pass}),
)
};
assert_eq!(pg("p1"), pg("p2"));
}
// dbt rejects a BigQuery target with no dataset and a service-account JSON
// carries none, so the descriptor has to supply it — and naming the missing
// field beats a downstream dbt error.
#[test]
fn bigquery_requires_a_dataset() {
let r = json!({"project_id": "p", "client_email": "e", "private_key": "k"});
let err = render_profile(
&KnownAdapter::Bigquery.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"prod",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap_err()
.to_string();
assert!(err.contains("profile.schema"), "{err}");
let p = render_profile(
&KnownAdapter::Bigquery.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"prod",
None,
Some("marts"),
std::path::Path::new("/tmp/p"),
)
.unwrap();
// dbt-bigquery's key is `dataset`, not `schema`.
assert!(p.yaml.contains(" dataset: \"marts\"\n"), "{}", p.yaml);
assert!(!p.yaml.contains(" schema:"));
assert_eq!(p.schema.as_deref(), Some("marts"));
}
// A SQL Server resource has a Postgres resource's shape, so guessing Postgres
// points dbt-postgres at port 1433 instead of producing the licensing error.
#[test]
fn ambiguous_host_resources_decline_rather_than_guess() {
let mssql = json!({"host": "h", "dbname": "d", "user": "u", "password": "p"});
assert_eq!(KnownAdapter::infer_from_resource(&mssql), None);
let pg = json!({"host": "h", "dbname": "d", "sslmode": "require"});
assert_eq!(
KnownAdapter::infer_from_resource(&pg),
Some(KnownAdapter::Postgres)
);
let sf = json!({"account_identifier": "acc", "database": "d"});
assert_eq!(
KnownAdapter::infer_from_resource(&sf),
Some(KnownAdapter::Snowflake)
);
}
// dbt-mysql has no database/schema split: one `schema` key is the database, so
// emitting the generic one too yields a profile with two.
#[test]
fn mysql_emits_exactly_one_schema_key() {
let r = json!({"host": "h", "dbname": "sales", "user": "u"});
let p = render_profile(
&KnownAdapter::Mysql.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"dev",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert_eq!(p.yaml.matches(" schema:").count(), 1);
assert!(p.yaml.contains(" schema: \"sales\"\n"));
assert!(p.yaml.contains(" port: 3306\n"));
assert_eq!(p.schema.as_deref(), Some("sales"));
}
// The profile name is the project's own `dbt_project.yml`, the target the
// descriptor's — both the author's text, and both were mapping keys.
#[test]
fn a_profile_name_or_target_cannot_open_a_sibling_key() {
let rendered = render_profile(
&KnownAdapter::Postgres.into(),
DbtEngine::DbtCore1x,
&serde_json::json!({"host": "h", "user": "u", "password": "p", "dbname": "d"}),
"prod # hidden",
"dev\n evil: yes",
None,
None,
std::path::Path::new("/tmp"),
)
.unwrap();
// Every occurrence is quoted, so neither the comment nor the newline is
// structure: the document still has exactly the keys we wrote.
assert!(
rendered.yaml.contains("\"prod # hidden\":"),
"{}",
rendered.yaml
);
assert!(
rendered.yaml.contains("\\n evil: yes"),
"{}",
rendered.yaml
);
let v: serde_yml::Value = serde_yml::from_str(&rendered.yaml).expect("valid yaml");
let profile = v.get("prod # hidden").expect("profile is one key");
assert!(profile.get("evil").is_none());
assert!(v.get("evil").is_none());
}
// A credential is attacker-influenced text. Unquoted, a `\n` or `"` closes the
// scalar and the rest reads as further profile keys — a different `host`,
// silently redirecting the run at another warehouse.
#[test]
fn credentials_cannot_break_out_of_their_scalar() {
let r = json!({"host": "h", "dbname": "d",
"password": "p\"\nhost: evil.example.com\n#"});
let p = render_profile(
&KnownAdapter::Postgres.into(),
DbtEngine::DbtCore1x,
&r,
"wm",
"dev",
None,
None,
std::path::Path::new("/tmp/p"),
)
.unwrap();
assert!(p
.yaml
.contains(r#"password: "p\"\nhost: evil.example.com\n#""#));
let parsed: serde_yml::Value = serde_yml::from_str(&p.yaml).unwrap();
assert_eq!(
parsed["wm"]["outputs"]["dev"]["host"].as_str(),
Some("h"),
"the injected host must not have won"
);
}
// A RUNTIME check, because one dbt executor serves every adapter. A refactor
// reaching for a bare `LICENSE_KEY_VALID` would silently let CE through, since
// the OSS variant initializes it to `true`.
#[test]
fn only_mssql_and_oracle_are_enterprise_gated() {
for a in [
KnownAdapter::Postgres,
KnownAdapter::Redshift,
KnownAdapter::Mysql,
KnownAdapter::Duckdb,
KnownAdapter::Clickhouse,
KnownAdapter::Snowflake,
KnownAdapter::Bigquery,
KnownAdapter::Databricks,
KnownAdapter::Salesforce,
] {
assert!(!a.requires_enterprise(), "{a:?}");
assert!(ensure_adapter_licensed(&a.into()).is_ok(), "{a:?}");
}
for (a, name) in [
(KnownAdapter::Mssql, "Microsoft SQL server"),
(KnownAdapter::OracleDB, "Oracle DB"),
] {
assert!(a.requires_enterprise(), "{a:?}");
match ensure_adapter_licensed(&a.into()) {
Ok(()) => assert!(
enterprise_licensed(),
"{a:?} was accepted without an enterprise license"
),
Err(e) => {
assert!(!enterprise_licensed(), "{a:?} rejected despite a license");
// Naming the adapter is the point: the user must not be left
// with a generic connection failure to diagnose.
assert_eq!(
e.to_string(),
format!("Bad request: {name} is only available with an enterprise license")
);
}
}
}
}
}