Skip to main content

operator/statement/
database_copy.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Shared selection and destination rules for database COPY and prepared exports.
16
17use std::collections::HashMap;
18
19use common_datasource::object_store::FILE_SCHEMA;
20#[cfg(windows)]
21use common_datasource::object_store::{FS_SCHEMA, parse_url};
22use common_stat::get_total_cpu_cores;
23use session::context::QueryContextRef;
24use snafu::{OptionExt, ResultExt, ensure};
25use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME};
26use table::TableRef;
27use table::metadata::TableType;
28use table::requests::CopyDatabaseRequest;
29use table::table_reference::TableReference;
30use url::Url;
31
32use crate::error::{self, Result};
33use crate::statement::StatementExecutor;
34
35fn is_directory_location(location: &str) -> bool {
36    if location.ends_with('/') {
37        return true;
38    }
39
40    #[cfg(windows)]
41    {
42        location.ends_with(std::path::MAIN_SEPARATOR)
43            && matches!(
44                parse_url(location),
45                Ok((schema, _, _)) if schema.eq_ignore_ascii_case(FS_SCHEMA)
46            )
47    }
48
49    #[cfg(not(windows))]
50    false
51}
52
53/// Get parallelism from options, default to total CPU cores.
54pub(crate) fn parse_parallelism_from_option_map(options: &HashMap<String, String>) -> usize {
55    options
56        .get("parallelism")
57        .and_then(|v| v.parse::<usize>().ok())
58        .unwrap_or_else(get_total_cpu_cores)
59        .max(1)
60}
61
62pub(crate) fn validate_database_directory(location: &str) -> Result<()> {
63    ensure!(
64        is_directory_location(location),
65        error::InvalidCopyDatabasePathSnafu { value: location }
66    );
67    #[cfg(windows)]
68    if common_datasource::object_store::handle_windows_path(location).is_some() {
69        return Ok(());
70    }
71    let parsed_directory = match Url::parse(location) {
72        Ok(url) => {
73            url.query().is_none() && url.fragment().is_none() && is_directory_location(url.path())
74        }
75        Err(_) => true,
76    };
77    ensure!(
78        parsed_directory,
79        error::InvalidCopyDatabasePathSnafu { value: location }
80    );
81    Ok(())
82}
83
84/// The writer key and its externally reported location, resolved together.
85pub(crate) struct DatabaseExportFile {
86    pub(crate) path: String,
87    pub(crate) location: String,
88}
89
90impl DatabaseExportFile {
91    pub(crate) fn new(directory: &str, name: &str, suffix: &str) -> Result<Self> {
92        let filename = format!("{name}{suffix}");
93        #[cfg(windows)]
94        if common_datasource::object_store::handle_windows_path(directory).is_some() {
95            return Ok(Self {
96                location: format!("{directory}{filename}"),
97                path: filename,
98            });
99        }
100        match Url::parse(directory) {
101            Ok(mut url) => {
102                url.path_segments_mut()
103                    .map_err(|_| error::InvalidCopyDatabasePathSnafu { value: directory }.build())?
104                    .pop_if_empty()
105                    .push(&filename);
106                // File URLs are decoded by the filesystem backend; object-store
107                // backends use the encoded URL path as their key.
108                let path = if url.scheme().eq_ignore_ascii_case(FILE_SCHEMA) {
109                    filename
110                } else {
111                    url.path()
112                        .rsplit('/')
113                        .next()
114                        .unwrap_or_default()
115                        .to_string()
116                };
117                Ok(Self {
118                    path,
119                    location: url.into(),
120                })
121            }
122            Err(url::ParseError::RelativeUrlWithoutBase) => Ok(Self {
123                location: format!("{directory}{filename}"),
124                path: filename,
125            }),
126            Err(source) => Err(source)
127                .context(common_datasource::error::InvalidUrlSnafu { url: directory })
128                .context(error::BuildBackendSnafu),
129        }
130    }
131}
132
133/// Resolve a listed writer key back to its table name and COPY input location.
134pub(crate) fn database_import_source(directory: &str, path: &str) -> Result<(String, String)> {
135    let mut filename = path.rsplit('/').next().unwrap_or(path).to_string();
136    let mut location = format!("{directory}{path}");
137    #[cfg(windows)]
138    let literal_path = common_datasource::object_store::handle_windows_path(directory).is_some();
139    #[cfg(not(windows))]
140    let literal_path = false;
141    if !literal_path && let Ok(mut url) = Url::parse(directory) {
142        if url.scheme().eq_ignore_ascii_case(FILE_SCHEMA) {
143            url.path_segments_mut()
144                .map_err(|_| error::InvalidCopyDatabasePathSnafu { value: directory }.build())?
145                .pop_if_empty()
146                .extend(path.split('/'));
147        } else {
148            // Listed object keys already contain the export URL's escaping.
149            url.set_path(&format!("{}{path}", url.path()));
150            filename = percent_encoding::percent_decode_str(&filename)
151                .decode_utf8()
152                .ok()
153                .context(error::InvalidTableNameSnafu { table_name: path })?
154                .into_owned();
155        }
156        location = url.into();
157    }
158    let table_name = filename
159        .rsplit_once('.')
160        .map(|(stem, _)| stem)
161        .filter(|stem| !stem.is_empty())
162        .context(error::InvalidTableNameSnafu { table_name: path })?
163        .to_string();
164    Ok((table_name, location))
165}
166
167impl StatementExecutor {
168    /// Capture each selected data table once. Views, temporary and Metric physical
169    /// tables do not have data outputs. `None` selects the whole schema.
170    pub async fn capture_database_export_tables(
171        &self,
172        req: &CopyDatabaseRequest,
173        names: Option<&[String]>,
174        ctx: &QueryContextRef,
175    ) -> Result<Vec<TableRef>> {
176        let mut names = match names {
177            Some(names) => names.to_vec(),
178            None => self
179                .catalog_manager
180                .table_names(&req.catalog_name, &req.schema_name, Some(ctx))
181                .await
182                .context(error::CatalogSnafu)?,
183        };
184        names.sort();
185        let mut tables = Vec::new();
186        for name in names {
187            let table = self
188                .get_table(&TableReference::full(
189                    &req.catalog_name,
190                    &req.schema_name,
191                    &name,
192                ))
193                .await?;
194            let info = table.table_info();
195            if table.table_type() == TableType::Base
196                && (info.meta.engine != METRIC_ENGINE_NAME
197                    || info
198                        .meta
199                        .options
200                        .extra_options
201                        .contains_key(LOGICAL_TABLE_METADATA_KEY))
202            {
203                tables.push(table);
204            }
205        }
206        Ok(tables)
207    }
208}
209
210#[cfg(test)]
211mod tests {
212    use super::*;
213
214    #[test]
215    fn directory_url_components_cannot_capture_output_names() {
216        for location in [
217            "file:///copy/fresh?attempt=/",
218            "file:///copy/fresh#attempt/",
219            "s3://bucket/fresh?attempt=/",
220            "s3://bucket/fresh#attempt/",
221            "file:///copy/fresh",
222        ] {
223            assert!(validate_database_directory(location).is_err(), "{location}");
224        }
225        for location in ["/copy/fresh/", "file:///copy/fresh/", "s3://bucket/fresh/"] {
226            validate_database_directory(location).unwrap();
227        }
228    }
229
230    #[cfg(windows)]
231    #[test]
232    fn windows_directory_names_are_literal_paths() {
233        for location in ["C:/copy/fresh#1/", r"C:\copy\fresh#1\"] {
234            validate_database_directory(location).unwrap();
235        }
236        assert!(validate_database_directory("C:/copy/fresh#1").is_err());
237    }
238
239    #[tokio::test]
240    async fn output_locations_resolve_to_writer_keys() {
241        use common_datasource::object_store::{LocalFileAccess, build_backend_for_write_with_path};
242
243        let dir = common_test_util::temp_dir::create_temp_dir("database_export_paths");
244        let access = LocalFileAccess::sandboxed(dir.path()).unwrap();
245        let file_url = Url::from_directory_path(dir.path()).unwrap().to_string();
246        let connection = HashMap::from([
247            ("region".into(), "us-east-1".into()),
248            ("access_key_id".into(), "test-key".into()),
249            ("secret_access_key".into(), "test-secret".into()),
250        ]);
251        for directory in [
252            format!("{}/", dir.path().display()),
253            file_url,
254            "s3://export-bucket/data/".into(),
255        ] {
256            for name in ["a#b", "a:b"] {
257                if cfg!(windows) && name.contains(':') && !directory.starts_with("s3:") {
258                    continue;
259                }
260                let file = DatabaseExportFile::new(&directory, name, ".parquet").unwrap();
261                let backend =
262                    build_backend_for_write_with_path(&file.location, &connection, &access)
263                        .await
264                        .unwrap();
265                assert_eq!(backend.object_path.as_deref(), Some(file.path.as_str()));
266                let (table_name, input_location) =
267                    database_import_source(&directory, &file.path).unwrap();
268                assert_eq!(table_name, name);
269                assert_eq!(input_location, file.location);
270                let (table_name, nested_location) =
271                    database_import_source(&directory, &format!("nested/{}", file.path)).unwrap();
272                assert_eq!(table_name, name);
273                assert_eq!(
274                    nested_location,
275                    file.location
276                        .replace(&directory, &format!("{directory}nested/"))
277                );
278                if !directory.starts_with("s3:") {
279                    backend
280                        .object_store
281                        .write(&file.path, "test")
282                        .await
283                        .unwrap();
284                    assert_eq!(
285                        std::fs::read(dir.path().join(format!("{name}.parquet"))).unwrap(),
286                        b"test"
287                    );
288                } else {
289                    assert_eq!(
290                        file.path,
291                        if name == "a#b" {
292                            "a%23b.parquet"
293                        } else {
294                            "a:b.parquet"
295                        }
296                    );
297                }
298            }
299        }
300    }
301
302    #[test]
303    fn test_parse_parallelism_from_option_map() {
304        let options = HashMap::new();
305        assert_eq!(
306            parse_parallelism_from_option_map(&options),
307            get_total_cpu_cores()
308        );
309
310        let options = HashMap::from([("parallelism".to_string(), "0".to_string())]);
311        assert_eq!(parse_parallelism_from_option_map(&options), 1);
312    }
313}