operator/statement/
database_copy.rs1use std::collections::HashMap;
18
19use common_datasource::object_store::FILE_SCHEMA;
20#[cfg(windows)]
21use common_datasource::object_store::{FS_SCHEMA, parse_url};
22use common_stat::get_total_cpu_cores;
23use session::context::QueryContextRef;
24use snafu::{OptionExt, ResultExt, ensure};
25use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME};
26use table::TableRef;
27use table::metadata::TableType;
28use table::requests::CopyDatabaseRequest;
29use table::table_reference::TableReference;
30use url::Url;
31
32use crate::error::{self, Result};
33use crate::statement::StatementExecutor;
34
35fn is_directory_location(location: &str) -> bool {
36 if location.ends_with('/') {
37 return true;
38 }
39
40 #[cfg(windows)]
41 {
42 location.ends_with(std::path::MAIN_SEPARATOR)
43 && matches!(
44 parse_url(location),
45 Ok((schema, _, _)) if schema.eq_ignore_ascii_case(FS_SCHEMA)
46 )
47 }
48
49 #[cfg(not(windows))]
50 false
51}
52
53pub(crate) fn parse_parallelism_from_option_map(options: &HashMap<String, String>) -> usize {
55 options
56 .get("parallelism")
57 .and_then(|v| v.parse::<usize>().ok())
58 .unwrap_or_else(get_total_cpu_cores)
59 .max(1)
60}
61
62pub(crate) fn validate_database_directory(location: &str) -> Result<()> {
63 ensure!(
64 is_directory_location(location),
65 error::InvalidCopyDatabasePathSnafu { value: location }
66 );
67 #[cfg(windows)]
68 if common_datasource::object_store::handle_windows_path(location).is_some() {
69 return Ok(());
70 }
71 let parsed_directory = match Url::parse(location) {
72 Ok(url) => {
73 url.query().is_none() && url.fragment().is_none() && is_directory_location(url.path())
74 }
75 Err(_) => true,
76 };
77 ensure!(
78 parsed_directory,
79 error::InvalidCopyDatabasePathSnafu { value: location }
80 );
81 Ok(())
82}
83
84pub(crate) struct DatabaseExportFile {
86 pub(crate) path: String,
87 pub(crate) location: String,
88}
89
90impl DatabaseExportFile {
91 pub(crate) fn new(directory: &str, name: &str, suffix: &str) -> Result<Self> {
92 let filename = format!("{name}{suffix}");
93 #[cfg(windows)]
94 if common_datasource::object_store::handle_windows_path(directory).is_some() {
95 return Ok(Self {
96 location: format!("{directory}{filename}"),
97 path: filename,
98 });
99 }
100 match Url::parse(directory) {
101 Ok(mut url) => {
102 url.path_segments_mut()
103 .map_err(|_| error::InvalidCopyDatabasePathSnafu { value: directory }.build())?
104 .pop_if_empty()
105 .push(&filename);
106 let path = if url.scheme().eq_ignore_ascii_case(FILE_SCHEMA) {
109 filename
110 } else {
111 url.path()
112 .rsplit('/')
113 .next()
114 .unwrap_or_default()
115 .to_string()
116 };
117 Ok(Self {
118 path,
119 location: url.into(),
120 })
121 }
122 Err(url::ParseError::RelativeUrlWithoutBase) => Ok(Self {
123 location: format!("{directory}{filename}"),
124 path: filename,
125 }),
126 Err(source) => Err(source)
127 .context(common_datasource::error::InvalidUrlSnafu { url: directory })
128 .context(error::BuildBackendSnafu),
129 }
130 }
131}
132
133pub(crate) fn database_import_source(directory: &str, path: &str) -> Result<(String, String)> {
135 let mut filename = path.rsplit('/').next().unwrap_or(path).to_string();
136 let mut location = format!("{directory}{path}");
137 #[cfg(windows)]
138 let literal_path = common_datasource::object_store::handle_windows_path(directory).is_some();
139 #[cfg(not(windows))]
140 let literal_path = false;
141 if !literal_path && let Ok(mut url) = Url::parse(directory) {
142 if url.scheme().eq_ignore_ascii_case(FILE_SCHEMA) {
143 url.path_segments_mut()
144 .map_err(|_| error::InvalidCopyDatabasePathSnafu { value: directory }.build())?
145 .pop_if_empty()
146 .extend(path.split('/'));
147 } else {
148 url.set_path(&format!("{}{path}", url.path()));
150 filename = percent_encoding::percent_decode_str(&filename)
151 .decode_utf8()
152 .ok()
153 .context(error::InvalidTableNameSnafu { table_name: path })?
154 .into_owned();
155 }
156 location = url.into();
157 }
158 let table_name = filename
159 .rsplit_once('.')
160 .map(|(stem, _)| stem)
161 .filter(|stem| !stem.is_empty())
162 .context(error::InvalidTableNameSnafu { table_name: path })?
163 .to_string();
164 Ok((table_name, location))
165}
166
167impl StatementExecutor {
168 pub async fn capture_database_export_tables(
171 &self,
172 req: &CopyDatabaseRequest,
173 names: Option<&[String]>,
174 ctx: &QueryContextRef,
175 ) -> Result<Vec<TableRef>> {
176 let mut names = match names {
177 Some(names) => names.to_vec(),
178 None => self
179 .catalog_manager
180 .table_names(&req.catalog_name, &req.schema_name, Some(ctx))
181 .await
182 .context(error::CatalogSnafu)?,
183 };
184 names.sort();
185 let mut tables = Vec::new();
186 for name in names {
187 let table = self
188 .get_table(&TableReference::full(
189 &req.catalog_name,
190 &req.schema_name,
191 &name,
192 ))
193 .await?;
194 let info = table.table_info();
195 if table.table_type() == TableType::Base
196 && (info.meta.engine != METRIC_ENGINE_NAME
197 || info
198 .meta
199 .options
200 .extra_options
201 .contains_key(LOGICAL_TABLE_METADATA_KEY))
202 {
203 tables.push(table);
204 }
205 }
206 Ok(tables)
207 }
208}
209
210#[cfg(test)]
211mod tests {
212 use super::*;
213
214 #[test]
215 fn directory_url_components_cannot_capture_output_names() {
216 for location in [
217 "file:///copy/fresh?attempt=/",
218 "file:///copy/fresh#attempt/",
219 "s3://bucket/fresh?attempt=/",
220 "s3://bucket/fresh#attempt/",
221 "file:///copy/fresh",
222 ] {
223 assert!(validate_database_directory(location).is_err(), "{location}");
224 }
225 for location in ["/copy/fresh/", "file:///copy/fresh/", "s3://bucket/fresh/"] {
226 validate_database_directory(location).unwrap();
227 }
228 }
229
230 #[cfg(windows)]
231 #[test]
232 fn windows_directory_names_are_literal_paths() {
233 for location in ["C:/copy/fresh#1/", r"C:\copy\fresh#1\"] {
234 validate_database_directory(location).unwrap();
235 }
236 assert!(validate_database_directory("C:/copy/fresh#1").is_err());
237 }
238
239 #[tokio::test]
240 async fn output_locations_resolve_to_writer_keys() {
241 use common_datasource::object_store::{LocalFileAccess, build_backend_for_write_with_path};
242
243 let dir = common_test_util::temp_dir::create_temp_dir("database_export_paths");
244 let access = LocalFileAccess::sandboxed(dir.path()).unwrap();
245 let file_url = Url::from_directory_path(dir.path()).unwrap().to_string();
246 let connection = HashMap::from([
247 ("region".into(), "us-east-1".into()),
248 ("access_key_id".into(), "test-key".into()),
249 ("secret_access_key".into(), "test-secret".into()),
250 ]);
251 for directory in [
252 format!("{}/", dir.path().display()),
253 file_url,
254 "s3://export-bucket/data/".into(),
255 ] {
256 for name in ["a#b", "a:b"] {
257 if cfg!(windows) && name.contains(':') && !directory.starts_with("s3:") {
258 continue;
259 }
260 let file = DatabaseExportFile::new(&directory, name, ".parquet").unwrap();
261 let backend =
262 build_backend_for_write_with_path(&file.location, &connection, &access)
263 .await
264 .unwrap();
265 assert_eq!(backend.object_path.as_deref(), Some(file.path.as_str()));
266 let (table_name, input_location) =
267 database_import_source(&directory, &file.path).unwrap();
268 assert_eq!(table_name, name);
269 assert_eq!(input_location, file.location);
270 let (table_name, nested_location) =
271 database_import_source(&directory, &format!("nested/{}", file.path)).unwrap();
272 assert_eq!(table_name, name);
273 assert_eq!(
274 nested_location,
275 file.location
276 .replace(&directory, &format!("{directory}nested/"))
277 );
278 if !directory.starts_with("s3:") {
279 backend
280 .object_store
281 .write(&file.path, "test")
282 .await
283 .unwrap();
284 assert_eq!(
285 std::fs::read(dir.path().join(format!("{name}.parquet"))).unwrap(),
286 b"test"
287 );
288 } else {
289 assert_eq!(
290 file.path,
291 if name == "a#b" {
292 "a%23b.parquet"
293 } else {
294 "a:b.parquet"
295 }
296 );
297 }
298 }
299 }
300 }
301
302 #[test]
303 fn test_parse_parallelism_from_option_map() {
304 let options = HashMap::new();
305 assert_eq!(
306 parse_parallelism_from_option_map(&options),
307 get_total_cpu_cores()
308 );
309
310 let options = HashMap::from([("parallelism".to_string(), "0".to_string())]);
311 assert_eq!(parse_parallelism_from_option_map(&options), 1);
312 }
313}