diff --git a/src/sql/src/statements/option_map.rs b/src/sql/src/statements/option_map.rs index 29808ae602d..e1dcc6900c0 100644 --- a/src/sql/src/statements/option_map.rs +++ b/src/sql/src/statements/option_map.rs @@ -19,11 +19,10 @@ use common_base::secrets::{ExposeSecret, ExposeSecretMut, SecretString}; use either::Either; use serde::Serialize; use sqlparser::ast::{Visit, VisitMut, Visitor, VisitorMut}; +use table::requests::{REDACTED_OPTION_VALUE, is_credential_option}; use crate::util::OptionValue; -const REDACTED_OPTIONS: [&str; 2] = ["access_key_id", "secret_access_key"]; - /// Options hashmap. #[derive(Clone, Debug, Default, Serialize)] pub struct OptionMap { @@ -36,7 +35,7 @@ impl OptionMap { pub fn new>(options: I) -> Self { let (secrets, options): (Vec<_>, Vec<_>) = options .into_iter() - .partition(|(k, _)| REDACTED_OPTIONS.contains(&k.as_str())); + .partition(|(k, _)| is_credential_option(k)); Self { options: options.into_iter().collect(), secrets: secrets @@ -62,7 +61,7 @@ impl OptionMap { } pub fn insert(&mut self, k: String, v: String) { - if REDACTED_OPTIONS.contains(&k.as_str()) { + if is_credential_option(&k) { self.secrets.insert(k, SecretString::new(Box::new(v))); } else { self.options.insert(k, v.into()); @@ -70,7 +69,7 @@ impl OptionMap { } pub fn insert_options(&mut self, key: &str, value: OptionValue) { - if REDACTED_OPTIONS.contains(&key) { + if is_credential_option(key) { self.secrets.insert( key.to_string(), SecretString::new(Box::new(value.to_string())), @@ -173,7 +172,7 @@ impl OptionMap { value: Either::Right(if complete { value.expose_secret() } else { - "******" + REDACTED_OPTION_VALUE }), }); options.chain(secrets) @@ -187,7 +186,7 @@ impl OptionMap { let secrets = self .secrets .keys() - .map(|k| (k.as_str(), Either::Right("******"))); + .map(|k| (k.as_str(), Either::Right(REDACTED_OPTION_VALUE))); std::iter::chain(options, secrets) } } diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs index f83af641323..289b8057d22 100644 --- a/src/table/src/requests.rs +++ b/src/table/src/requests.rs @@ -14,7 +14,7 @@ //! Table and TableEngine requests -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt; use std::str::FromStr; @@ -316,6 +316,23 @@ impl TableOptions { } } +/// Object store credential keys accepted by [`validate_table_option`]. +const CREDENTIAL_OPTION_KEYS: [&str; 4] = [ + "access_key_id", + "secret_access_key", + "session_token", + "access_key_secret", +]; + +/// Placeholder rendered instead of a credential value. +pub const REDACTED_OPTION_VALUE: &str = "******"; + +pub fn is_credential_option(key: &str) -> bool { + CREDENTIAL_OPTION_KEYS.contains(&key) +} + +/// Renders the options for users, e.g. `information_schema.tables.create_options`, +/// so credential values are masked. impl fmt::Display for TableOptions { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut key_vals = vec![]; @@ -331,7 +348,14 @@ impl fmt::Display for TableOptions { key_vals.push(format!("{}={}", SKIP_WAL_KEY, self.skip_wal)); } - for (k, v) in &self.extra_options { + // Sort by key so `create_options` is stable across queries; `HashMap` order is not. + let extra_options: BTreeMap<_, _> = self.extra_options.iter().collect(); + for (k, v) in extra_options { + let v = if is_credential_option(k) { + REDACTED_OPTION_VALUE + } else { + v + }; key_vals.push(format!("{}={}", k, v)); } diff --git a/tests/cases/standalone/external_table_credentials.result b/tests/cases/standalone/external_table_credentials.result new file mode 100644 index 00000000000..822c0bdf164 --- /dev/null +++ b/tests/cases/standalone/external_table_credentials.result @@ -0,0 +1,66 @@ +CREATE TABLE external_table_credentials_source ( + ts TIMESTAMP TIME INDEX, + val DOUBLE +); + +Affected Rows: 0 + +INSERT INTO external_table_credentials_source VALUES (1, 1.0); + +Affected Rows: 1 + +COPY external_table_credentials_source TO 'external_table_credentials/table.parquet'; + +Affected Rows: 1 + +CREATE EXTERNAL TABLE external_table_credentials +WITH ( + location = 'external_table_credentials/table.parquet', + format = 'parquet', + access_key_id = 'AKID_PLAIN', + secret_access_key = 'SECRET_PLAIN', + session_token = 'TOKEN_PLAIN', + access_key_secret = 'OSS_SECRET_PLAIN' +); + +Affected Rows: 0 + +SHOW CREATE TABLE external_table_credentials; + ++----------------------------+--------------------------------------------------------------------+ +| Table | Create Table | ++----------------------------+--------------------------------------------------------------------+ +| external_table_credentials | CREATE EXTERNAL TABLE IF NOT EXISTS "external_table_credentials" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "val" DOUBLE NULL, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=file | +| | WITH( | +| | format = 'parquet', | +| | location = 'external_table_credentials/table.parquet', | +| | access_key_id = '******', | +| | access_key_secret = '******', | +| | secret_access_key = '******', | +| | session_token = '******' | +| | ) | ++----------------------------+--------------------------------------------------------------------+ + +-- SQLNESS REPLACE __private\.file_table_meta=\S+ FILE_TABLE_META +SELECT create_options FROM information_schema.tables WHERE table_name = 'external_table_credentials'; + ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| create_options | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| FILE_TABLE_META access_key_id=****** access_key_secret=****** format=parquet location=external_table_credentials/table.parquet secret_access_key=****** session_token=****** | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +DROP TABLE external_table_credentials; + +Affected Rows: 0 + +DROP TABLE external_table_credentials_source; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/external_table_credentials.sql b/tests/cases/standalone/external_table_credentials.sql new file mode 100644 index 00000000000..5d86fa161d5 --- /dev/null +++ b/tests/cases/standalone/external_table_credentials.sql @@ -0,0 +1,27 @@ +CREATE TABLE external_table_credentials_source ( + ts TIMESTAMP TIME INDEX, + val DOUBLE +); + +INSERT INTO external_table_credentials_source VALUES (1, 1.0); + +COPY external_table_credentials_source TO 'external_table_credentials/table.parquet'; + +CREATE EXTERNAL TABLE external_table_credentials +WITH ( + location = 'external_table_credentials/table.parquet', + format = 'parquet', + access_key_id = 'AKID_PLAIN', + secret_access_key = 'SECRET_PLAIN', + session_token = 'TOKEN_PLAIN', + access_key_secret = 'OSS_SECRET_PLAIN' +); + +SHOW CREATE TABLE external_table_credentials; + +-- SQLNESS REPLACE __private\.file_table_meta=\S+ FILE_TABLE_META +SELECT create_options FROM information_schema.tables WHERE table_name = 'external_table_credentials'; + +DROP TABLE external_table_credentials; + +DROP TABLE external_table_credentials_source;