cli/data/export_v2/
data.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use common_base::secrets::{ExposeSecret, SecretString};
16use common_telemetry::info;
17use object_store::util::{join_path, normalize_path};
18use snafu::ResultExt;
19use url::Url;
20
21use crate::common::ObjectStoreConfig;
22use crate::data::export_v2::error::{DatabaseSnafu, InvalidUriSnafu, Result, UrlParseSnafu};
23use crate::data::export_v2::manifest::{DataFormat, TimeRange};
24use crate::data::path::data_dir_for_schema_chunk;
25use crate::data::snapshot_storage::StorageScheme;
26use crate::data::sql::{escape_sql_identifier, escape_sql_literal};
27use crate::database::DatabaseClient;
28
29pub(super) struct CopyOptions {
30    pub(super) format: DataFormat,
31    pub(super) time_range: TimeRange,
32    pub(super) parallelism: usize,
33    pub(crate) experimental_metric_export: bool,
34}
35
36pub(super) struct CopyTarget {
37    pub(super) location: String,
38    pub(super) connection: String,
39    secrets: Vec<Option<String>>,
40}
41
42pub(crate) struct CopySource {
43    pub(crate) location: String,
44    pub(crate) connection: String,
45    secrets: Vec<Option<String>>,
46}
47
48impl CopyTarget {
49    fn mask_sql(&self, sql: &str) -> String {
50        mask_secrets(sql, &self.secrets)
51    }
52}
53
54impl CopySource {
55    fn mask_sql(&self, sql: &str) -> String {
56        mask_secrets(sql, &self.secrets)
57    }
58}
59
60pub(super) fn build_copy_target(
61    snapshot_uri: &str,
62    storage: &ObjectStoreConfig,
63    schema: &str,
64    chunk_id: u32,
65) -> Result<CopyTarget> {
66    let location = build_copy_location(snapshot_uri, storage, schema, chunk_id)?;
67    Ok(CopyTarget {
68        location: location.location,
69        connection: location.connection,
70        secrets: location.secrets,
71    })
72}
73
74pub(crate) fn build_copy_source(
75    snapshot_uri: &str,
76    storage: &ObjectStoreConfig,
77    schema: &str,
78    chunk_id: u32,
79) -> Result<CopySource> {
80    let location = build_copy_location(snapshot_uri, storage, schema, chunk_id)?;
81    Ok(CopySource {
82        location: location.location,
83        connection: location.connection,
84        secrets: location.secrets,
85    })
86}
87
88struct CopyLocation {
89    location: String,
90    connection: String,
91    secrets: Vec<Option<String>>,
92}
93
94fn build_copy_location(
95    snapshot_uri: &str,
96    storage: &ObjectStoreConfig,
97    schema: &str,
98    chunk_id: u32,
99) -> Result<CopyLocation> {
100    let url = Url::parse(snapshot_uri).context(UrlParseSnafu)?;
101    let scheme = StorageScheme::from_uri(snapshot_uri)?;
102    let suffix = data_dir_for_schema_chunk(schema, chunk_id);
103
104    match scheme {
105        StorageScheme::File => {
106            let root = url.to_file_path().map_err(|_| {
107                InvalidUriSnafu {
108                    uri: snapshot_uri,
109                    reason: "file:// URI must use an absolute path like file:///tmp/backup",
110                }
111                .build()
112            })?;
113            let location = normalize_path(&format!("{}/{}", root.to_string_lossy(), suffix));
114            Ok(CopyLocation {
115                location,
116                connection: String::new(),
117                secrets: Vec::new(),
118            })
119        }
120        StorageScheme::S3 => {
121            let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
122            let location = format!("s3://{}/{}", bucket, join_root(&root, &suffix));
123            let (connection, secrets) = build_s3_connection(storage);
124            Ok(CopyLocation {
125                location,
126                connection,
127                secrets,
128            })
129        }
130        StorageScheme::Oss => {
131            let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
132            let location = format!("oss://{}/{}", bucket, join_root(&root, &suffix));
133            let (connection, secrets) = build_oss_connection(storage);
134            Ok(CopyLocation {
135                location,
136                connection,
137                secrets,
138            })
139        }
140        StorageScheme::Gcs => {
141            let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
142            let location = format!("gcs://{}/{}", bucket, join_root(&root, &suffix));
143            let (connection, secrets) = build_gcs_connection(storage, snapshot_uri)?;
144            Ok(CopyLocation {
145                location,
146                connection,
147                secrets,
148            })
149        }
150        StorageScheme::Azblob => {
151            let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
152            let location = format!("azblob://{}/{}", bucket, join_root(&root, &suffix));
153            let (connection, secrets) = build_azblob_connection(storage);
154            Ok(CopyLocation {
155                location,
156                connection,
157                secrets,
158            })
159        }
160    }
161}
162
163pub(super) async fn execute_copy_database(
164    database_client: &DatabaseClient,
165    catalog: &str,
166    schema: &str,
167    target: &CopyTarget,
168    options: &CopyOptions,
169) -> Result<()> {
170    let with_options = build_with_options(options);
171    let sql = format!(
172        r#"COPY DATABASE "{}"."{}" TO '{}' WITH ({}){};"#,
173        escape_sql_identifier(catalog),
174        escape_sql_identifier(schema),
175        escape_sql_literal(&target.location),
176        with_options,
177        target.connection
178    );
179    let safe_sql = target.mask_sql(&sql);
180    info!("Executing sql: {}", safe_sql);
181    database_client
182        .sql_in_public(&sql)
183        .await
184        .context(DatabaseSnafu)?;
185    Ok(())
186}
187
188pub(crate) async fn execute_copy_database_from(
189    database_client: &DatabaseClient,
190    catalog: &str,
191    schema: &str,
192    source: &CopySource,
193    format: DataFormat,
194    packed: bool,
195) -> Result<()> {
196    let sql = format!(
197        r#"COPY DATABASE "{}"."{}" FROM '{}' WITH (FORMAT='{}'{}){};"#,
198        escape_sql_identifier(catalog),
199        escape_sql_identifier(schema),
200        escape_sql_literal(&source.location),
201        format,
202        if packed {
203            ", metric_data_layout='packed'"
204        } else {
205            ""
206        },
207        source.connection
208    );
209    let safe_sql = source.mask_sql(&sql);
210    info!("Executing sql: {}", safe_sql);
211    database_client
212        .sql_in_public(&sql)
213        .await
214        .context(DatabaseSnafu)?;
215    Ok(())
216}
217
218fn build_with_options(options: &CopyOptions) -> String {
219    let mut parts = vec![format!("FORMAT='{}'", options.format)];
220    if options.experimental_metric_export {
221        parts.push("experimental_metric_export='true'".to_string());
222    }
223    if let Some(start) = options.time_range.start {
224        parts.push(format!(
225            "START_TIME='{}'",
226            escape_sql_literal(&start.to_rfc3339())
227        ));
228    }
229    if let Some(end) = options.time_range.end {
230        parts.push(format!(
231            "END_TIME='{}'",
232            escape_sql_literal(&end.to_rfc3339())
233        ));
234    }
235    parts.push(format!("PARALLELISM={}", options.parallelism));
236    parts.join(", ")
237}
238
239fn extract_bucket_root(url: &Url, snapshot_uri: &str) -> Result<(String, String)> {
240    let bucket = url.host_str().unwrap_or("").to_string();
241    if bucket.is_empty() {
242        return InvalidUriSnafu {
243            uri: snapshot_uri,
244            reason: "URI must include bucket/container in host",
245        }
246        .fail();
247    }
248    let root = url
249        .path()
250        .trim_start_matches('/')
251        .trim_end_matches('/')
252        .to_string();
253    Ok((bucket, root))
254}
255
256fn join_root(root: &str, suffix: &str) -> String {
257    join_path(root, suffix).trim_start_matches('/').to_string()
258}
259
260fn build_s3_connection(storage: &ObjectStoreConfig) -> (String, Vec<Option<String>>) {
261    let access_key_id = expose_optional_secret(&storage.s3.s3_access_key_id);
262    let secret_access_key = expose_optional_secret(&storage.s3.s3_secret_access_key);
263
264    let mut options = Vec::new();
265    if let Some(access_key_id) = &access_key_id {
266        options.push(format!(
267            "ACCESS_KEY_ID='{}'",
268            escape_sql_literal(access_key_id)
269        ));
270    }
271    if let Some(secret_access_key) = &secret_access_key {
272        options.push(format!(
273            "SECRET_ACCESS_KEY='{}'",
274            escape_sql_literal(secret_access_key)
275        ));
276    }
277    if let Some(region) = &storage.s3.s3_region {
278        options.push(format!("REGION='{}'", escape_sql_literal(region)));
279    }
280    if let Some(endpoint) = &storage.s3.s3_endpoint {
281        options.push(format!("ENDPOINT='{}'", escape_sql_literal(endpoint)));
282    }
283
284    let secrets = vec![access_key_id, secret_access_key];
285    let connection = if options.is_empty() {
286        String::new()
287    } else {
288        format!(" CONNECTION ({})", options.join(", "))
289    };
290    (connection, secrets)
291}
292
293fn build_oss_connection(storage: &ObjectStoreConfig) -> (String, Vec<Option<String>>) {
294    let access_key_id = expose_optional_secret(&storage.oss.oss_access_key_id);
295    let access_key_secret = expose_optional_secret(&storage.oss.oss_access_key_secret);
296
297    let mut options = Vec::new();
298    if let Some(access_key_id) = &access_key_id {
299        options.push(format!(
300            "ACCESS_KEY_ID='{}'",
301            escape_sql_literal(access_key_id)
302        ));
303    }
304    if let Some(access_key_secret) = &access_key_secret {
305        options.push(format!(
306            "ACCESS_KEY_SECRET='{}'",
307            escape_sql_literal(access_key_secret)
308        ));
309    }
310    if !storage.oss.oss_endpoint.is_empty() {
311        options.push(format!(
312            "ENDPOINT='{}'",
313            escape_sql_literal(&storage.oss.oss_endpoint)
314        ));
315    }
316
317    let secrets = vec![access_key_id, access_key_secret];
318    let connection = if options.is_empty() {
319        String::new()
320    } else {
321        format!(" CONNECTION ({})", options.join(", "))
322    };
323    (connection, secrets)
324}
325
326fn build_gcs_connection(
327    storage: &ObjectStoreConfig,
328    snapshot_uri: &str,
329) -> Result<(String, Vec<Option<String>>)> {
330    let credential_path = expose_optional_secret(&storage.gcs.gcs_credential_path);
331    let credential = expose_optional_secret(&storage.gcs.gcs_credential);
332
333    if credential.is_none() && credential_path.is_some() {
334        return InvalidUriSnafu {
335            uri: snapshot_uri,
336            reason: "gcs_credential_path is not supported for server-side COPY; provide gcs_credential or rely on server-side ADC",
337        }
338        .fail();
339    }
340
341    let mut options = Vec::new();
342    if let Some(credential) = &credential {
343        options.push(format!("CREDENTIAL='{}'", escape_sql_literal(credential)));
344    }
345    if !storage.gcs.gcs_scope.is_empty() {
346        options.push(format!(
347            "SCOPE='{}'",
348            escape_sql_literal(&storage.gcs.gcs_scope)
349        ));
350    }
351    if !storage.gcs.gcs_endpoint.is_empty() {
352        options.push(format!(
353            "ENDPOINT='{}'",
354            escape_sql_literal(&storage.gcs.gcs_endpoint)
355        ));
356    }
357
358    let connection = if options.is_empty() {
359        String::new()
360    } else {
361        format!(" CONNECTION ({})", options.join(", "))
362    };
363    let secrets = vec![credential_path, credential];
364    Ok((connection, secrets))
365}
366
367fn build_azblob_connection(storage: &ObjectStoreConfig) -> (String, Vec<Option<String>>) {
368    let account_name = expose_optional_secret(&storage.azblob.azblob_account_name);
369    let account_key = expose_optional_secret(&storage.azblob.azblob_account_key);
370    let sas_token = storage.azblob.azblob_sas_token.clone();
371
372    let mut options = Vec::new();
373    if let Some(account_name) = &account_name {
374        options.push(format!(
375            "ACCOUNT_NAME='{}'",
376            escape_sql_literal(account_name)
377        ));
378    }
379    if let Some(account_key) = &account_key {
380        options.push(format!("ACCOUNT_KEY='{}'", escape_sql_literal(account_key)));
381    }
382    if let Some(sas_token) = &sas_token {
383        options.push(format!("SAS_TOKEN='{}'", escape_sql_literal(sas_token)));
384    }
385    if !storage.azblob.azblob_endpoint.is_empty() {
386        options.push(format!(
387            "ENDPOINT='{}'",
388            escape_sql_literal(&storage.azblob.azblob_endpoint)
389        ));
390    }
391
392    let secrets = vec![account_name, account_key, sas_token];
393    let connection = if options.is_empty() {
394        String::new()
395    } else {
396        format!(" CONNECTION ({})", options.join(", "))
397    };
398    (connection, secrets)
399}
400
401fn expose_optional_secret(secret: &Option<SecretString>) -> Option<String> {
402    secret.as_ref().map(|s| s.expose_secret().to_owned())
403}
404
405fn mask_secrets(sql: &str, secrets: &[Option<String>]) -> String {
406    let mut masked = sql.to_string();
407    for secret in secrets {
408        if let Some(secret) = secret
409            && !secret.is_empty()
410        {
411            let escaped = escape_sql_literal(secret);
412            if escaped != *secret {
413                masked = masked.replace(&escaped, "[REDACTED]");
414            }
415            masked = masked.replace(secret, "[REDACTED]");
416        }
417    }
418    masked
419}
420
421#[cfg(test)]
422mod tests {
423    use common_base::secrets::SecretString;
424    use common_test_util::temp_dir::create_temp_dir;
425
426    use super::*;
427    use crate::common::{PrefixedAzblobConnection, PrefixedGcsConnection, PrefixedOssConnection};
428
429    #[test]
430    fn test_build_oss_connection_includes_endpoint() {
431        let storage = ObjectStoreConfig {
432            oss: PrefixedOssConnection {
433                oss_endpoint: "https://oss.example.com".to_string(),
434                oss_access_key_id: Some(SecretString::from("key_id".to_string())),
435                oss_access_key_secret: Some(SecretString::from("key_secret".to_string())),
436                ..Default::default()
437            },
438            ..Default::default()
439        };
440
441        let (connection, _) = build_oss_connection(&storage);
442        assert!(connection.contains("ENDPOINT='https://oss.example.com'"));
443    }
444
445    #[test]
446    fn test_build_gcs_connection_uses_scope_and_inline_credential() {
447        let storage = ObjectStoreConfig {
448            gcs: PrefixedGcsConnection {
449                gcs_scope: "scope-a".to_string(),
450                gcs_endpoint: "https://storage.googleapis.com".to_string(),
451                gcs_credential: Some(SecretString::from("credential-json".to_string())),
452                ..Default::default()
453            },
454            ..Default::default()
455        };
456
457        let (connection, _) = build_gcs_connection(&storage, "gcs://bucket/root").unwrap();
458        assert!(connection.contains("CREDENTIAL='credential-json'"));
459        assert!(connection.contains("SCOPE='scope-a'"));
460        assert!(connection.contains("ENDPOINT='https://storage.googleapis.com'"));
461        assert!(!connection.contains("CREDENTIAL_PATH"));
462    }
463
464    #[test]
465    fn test_build_gcs_connection_rejects_credential_path_only() {
466        let storage = ObjectStoreConfig {
467            gcs: PrefixedGcsConnection {
468                gcs_scope: "scope-a".to_string(),
469                gcs_credential_path: Some(SecretString::from("/tmp/creds.json".to_string())),
470                ..Default::default()
471            },
472            ..Default::default()
473        };
474
475        let error = build_gcs_connection(&storage, "gcs://bucket/root")
476            .expect_err("credential_path-only should be rejected")
477            .to_string();
478        assert!(error.contains("gcs_credential_path is not supported"));
479    }
480
481    #[test]
482    fn test_build_azblob_connection_includes_endpoint() {
483        let storage = ObjectStoreConfig {
484            azblob: PrefixedAzblobConnection {
485                azblob_account_name: Some(SecretString::from("account".to_string())),
486                azblob_account_key: Some(SecretString::from("key".to_string())),
487                azblob_endpoint: "https://blob.example.com".to_string(),
488                ..Default::default()
489            },
490            ..Default::default()
491        };
492
493        let (connection, _) = build_azblob_connection(&storage);
494        assert!(connection.contains("ENDPOINT='https://blob.example.com'"));
495    }
496
497    #[test]
498    fn test_build_azblob_connection_redacts_sas_token() {
499        let storage = ObjectStoreConfig {
500            azblob: PrefixedAzblobConnection {
501                azblob_account_name: Some(SecretString::from("account".to_string())),
502                azblob_account_key: Some(SecretString::from("key".to_string())),
503                azblob_sas_token: Some("sig=secret-token".to_string()),
504                ..Default::default()
505            },
506            ..Default::default()
507        };
508
509        let (connection, secrets) = build_azblob_connection(&storage);
510        let masked = mask_secrets(&connection, &secrets);
511
512        assert!(connection.contains("SAS_TOKEN='sig=secret-token'"));
513        assert!(masked.contains("SAS_TOKEN='[REDACTED]'"));
514        assert!(!masked.contains("sig=secret-token"));
515    }
516
517    #[test]
518    fn test_mask_secrets_redacts_sql_escaped_literals() {
519        let sql =
520            "COPY DATABASE \"greptime\".\"public\" TO 's3://bucket' CONNECTION (SECRET='ab''cd');";
521        let masked = mask_secrets(sql, &[Some("ab'cd".to_string())]);
522
523        assert!(!masked.contains("ab'cd"));
524        assert!(!masked.contains("ab''cd"));
525        assert!(masked.contains("SECRET='[REDACTED]'"));
526    }
527
528    #[test]
529    fn test_build_copy_target_decodes_file_uri_path() {
530        let storage = ObjectStoreConfig::default();
531        let snapshot_root = create_temp_dir("my backup");
532        let snapshot_uri = Url::from_file_path(snapshot_root.path())
533            .expect("absolute platform path should convert to file:// URI")
534            .to_string();
535        let expected = normalize_path(&format!(
536            "{}/{}",
537            snapshot_root.path().to_string_lossy(),
538            data_dir_for_schema_chunk("public", 7)
539        ));
540        let target = build_copy_target(&snapshot_uri, &storage, "public", 7)
541            .expect("file:// copy target should be built");
542
543        assert!(snapshot_uri.contains("%20"));
544        assert!(!target.location.contains("%20"));
545        assert!(target.location.contains("my backup"));
546        assert_eq!(target.location, expected);
547    }
548}
cli/data/export_v2/data.rs

cli/data/export_v2/
data.rs
