1use std::str::FromStr;
16use std::sync::Arc;
17
18use client::{Output, OutputData, OutputMeta};
19use common_catalog::format_full_table_name;
20use common_datasource::file_format::Format;
21use common_datasource::lister::{Lister, Source};
22use common_datasource::object_store::{LocalFileAccess, build_backend, build_backend_for_write};
23use common_telemetry::{debug, error, info, tracing};
24use futures::future::try_join_all;
25use object_store::Entry;
26use regex::Regex;
27use session::context::QueryContextRef;
28use snafu::ResultExt;
29use table::requests::{CopyDatabaseRequest, CopyDirection, CopyTableRequest};
30use tokio::sync::Semaphore;
31
32use crate::error;
33use crate::statement::StatementExecutor;
34use crate::statement::database_copy::{
35 DatabaseExportFile, database_import_source, parse_parallelism_from_option_map,
36 validate_database_directory,
37};
38
39pub(crate) const COPY_DATABASE_TIME_START_KEY: &str = "start_time";
40pub(crate) const COPY_DATABASE_TIME_END_KEY: &str = "end_time";
41pub(crate) const CONTINUE_ON_ERROR_KEY: &str = "continue_on_error";
42
43impl StatementExecutor {
44 #[tracing::instrument(skip_all)]
45 pub(crate) async fn copy_database_to(
46 &self,
47 req: CopyDatabaseRequest,
48 ctx: QueryContextRef,
49 ) -> error::Result<Output> {
50 validate_database_directory(&req.location)?;
51 build_backend_for_write(&req.location, &req.connection, &self.local_file_access)
52 .await
53 .context(error::BuildBackendSnafu)?;
54
55 let parallelism = parse_parallelism_from_option_map(&req.with);
56 info!(
57 "Copy database {}.{} to dir: {}, time: {:?}, parallelism: {}",
58 req.catalog_name, req.schema_name, req.location, req.time_range, parallelism
59 );
60 let tables = self
61 .capture_database_export_tables(&req, None, &ctx)
62 .await?;
63 let num_tables = tables.len();
64
65 let suffix = Format::try_from(&req.with)
66 .context(error::ParseFileFormatSnafu)?
67 .suffix();
68
69 let mut tasks = Vec::with_capacity(num_tables);
70 let semaphore = Arc::new(Semaphore::new(parallelism));
71
72 for (i, table) in tables.into_iter().enumerate() {
73 let table_name = table.table_info().name.clone();
74 let semaphore_moved = semaphore.clone();
75 let table_file = DatabaseExportFile::new(&req.location, &table_name, suffix)?.location;
76 let table_no = i + 1;
77 let moved_ctx = ctx.clone();
78 let full_table_name =
79 format_full_table_name(&req.catalog_name, &req.schema_name, &table_name);
80 let copy_table_req = CopyTableRequest {
81 catalog_name: req.catalog_name.clone(),
82 schema_name: req.schema_name.clone(),
83 table_name,
84 location: table_file.clone(),
85 with: req.with.clone(),
86 connection: req.connection.clone(),
87 pattern: None,
88 direction: CopyDirection::Export,
89 timestamp_range: req.time_range,
90 limit: None,
91 };
92
93 tasks.push(async move {
94 let _permit = semaphore_moved.acquire().await.unwrap();
95 info!(
96 "Copy table({}/{}): {} to {}",
97 table_no, num_tables, full_table_name, table_file
98 );
99 self.copy_captured_table_to(table, copy_table_req, moved_ctx)
100 .await
101 });
102 }
103
104 let results = try_join_all(tasks).await?;
105 let exported_rows = results.into_iter().sum();
106 Ok(Output::new_with_affected_rows(exported_rows))
107 }
108
109 #[tracing::instrument(skip_all)]
111 pub(crate) async fn copy_database_from(
112 &self,
113 req: CopyDatabaseRequest,
114 ctx: QueryContextRef,
115 ) -> error::Result<Output> {
116 validate_database_directory(&req.location)?;
117
118 let parallelism = parse_parallelism_from_option_map(&req.with);
119 info!(
120 "Copy database {}.{} from dir: {}, time: {:?}, parallelism: {}",
121 req.catalog_name, req.schema_name, req.location, req.time_range, parallelism
122 );
123 let suffix = Format::try_from(&req.with)
124 .context(error::ParseFileFormatSnafu)?
125 .suffix();
126
127 let entries = list_files_to_copy(&req, suffix, &self.local_file_access).await?;
128
129 let continue_on_error = req
130 .with
131 .get(CONTINUE_ON_ERROR_KEY)
132 .and_then(|v| bool::from_str(v).ok())
133 .unwrap_or(false);
134
135 let mut tasks = Vec::with_capacity(entries.len());
136 let semaphore = Arc::new(Semaphore::new(parallelism));
137
138 for e in entries {
139 let (table_name, location) = match database_import_source(&req.location, e.path()) {
140 Ok(source) => source,
141 Err(err) => {
142 if continue_on_error {
143 error!(err; "Failed to import table from file: {:?}", e);
144 continue;
145 } else {
146 return Err(err);
147 }
148 }
149 };
150
151 let req = CopyTableRequest {
152 catalog_name: req.catalog_name.clone(),
153 schema_name: req.schema_name.clone(),
154 table_name: table_name.clone(),
155 location,
156 with: req.with.clone(),
157 connection: req.connection.clone(),
158 pattern: None,
159 direction: CopyDirection::Import,
160 timestamp_range: None,
161 limit: None,
162 };
163 let moved_ctx = ctx.clone();
164 let moved_table_name = table_name.clone();
165 let moved_semaphore = semaphore.clone();
166 tasks.push(async move {
167 let _permit = moved_semaphore.acquire().await.unwrap();
168 debug!("Copy table, arg: {:?}", req);
169 match self.copy_table_from(req, moved_ctx).await {
170 Ok(o) => {
171 let (rows, cost) = o.extract_rows_and_cost();
172 Ok((rows, cost))
173 }
174 Err(err) => {
175 if continue_on_error {
176 error!(err; "Failed to import file to table: {}", moved_table_name);
177 Ok((0, 0))
178 } else {
179 Err(err)
180 }
181 }
182 }
183 });
184 }
185
186 let results = try_join_all(tasks).await?;
187 let (rows_inserted, insert_cost) = results
188 .into_iter()
189 .fold((0, 0), |(acc_rows, acc_cost), (rows, cost)| {
190 (acc_rows + rows, acc_cost + cost)
191 });
192
193 Ok(Output::new(
194 OutputData::AffectedRows(rows_inserted),
195 OutputMeta::new_with_cost(insert_cost),
196 ))
197 }
198}
199
200async fn list_files_to_copy(
202 req: &CopyDatabaseRequest,
203 suffix: &str,
204 local_file_access: &LocalFileAccess,
205) -> error::Result<Vec<Entry>> {
206 let object_store = build_backend(&req.location, &req.connection, local_file_access)
207 .await
208 .context(error::BuildBackendSnafu)?;
209
210 let pattern = Regex::try_from(format!(".*{}", suffix)).context(error::BuildRegexSnafu)?;
211 let lister = Lister::new(
212 object_store.clone(),
213 Source::Dir,
214 "/".to_string(),
215 Some(pattern),
216 );
217 lister.list().await.context(error::ListObjectsSnafu)
218}
219
220#[cfg(test)]
221mod tests {
222 use std::collections::HashSet;
223
224 use common_datasource::object_store::LocalFileAccess;
225 use object_store::ObjectStore;
226 use object_store::services::Fs;
227 use object_store::util::normalize_dir;
228 #[cfg(not(windows))]
229 use path_slash::PathExt;
230 use table::requests::CopyDatabaseRequest;
231
232 use crate::statement::copy_database::list_files_to_copy;
233 use crate::statement::database_copy::database_import_source;
234
235 #[tokio::test]
236 async fn test_list_files_and_parse_table_name() {
237 let dir = common_test_util::temp_dir::create_temp_dir("test_list_files_to_copy");
238 let store_dir = normalize_dir(dir.path().to_str().unwrap());
239 let builder = Fs::default().root(&store_dir);
240 let object_store = ObjectStore::new(builder).unwrap();
241 object_store.write("a.parquet", "").await.unwrap();
242 object_store.write("b.parquet", "").await.unwrap();
243 object_store.write("c.csv", "").await.unwrap();
244 object_store.write("d", "").await.unwrap();
245 object_store.write("e.f.parquet", "").await.unwrap();
246
247 #[cfg(not(windows))]
248 let location = normalize_dir(&dir.path().to_slash().unwrap());
249 #[cfg(windows)]
250 let location = format!("{}\\", dir.path().display());
251 let request = CopyDatabaseRequest {
252 catalog_name: "catalog_0".to_string(),
253 schema_name: "schema_0".to_string(),
254 location,
255 with: [("FORMAT".to_string(), "parquet".to_string())]
256 .into_iter()
257 .collect(),
258 connection: Default::default(),
259 time_range: None,
260 };
261 let local_file_access = LocalFileAccess::sandboxed(dir.path()).unwrap();
262 let listed = list_files_to_copy(&request, ".parquet", &local_file_access)
263 .await
264 .unwrap()
265 .into_iter()
266 .map(|e| {
267 database_import_source(&request.location, e.path())
268 .unwrap()
269 .0
270 })
271 .collect::<HashSet<_>>();
272
273 assert_eq!(
274 ["a".to_string(), "b".to_string(), "e.f".to_string()]
275 .into_iter()
276 .collect::<HashSet<_>>(),
277 listed
278 );
279 }
280}