Skip to main content

operator/statement/
copy_database.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::str::FromStr;
16use std::sync::Arc;
17
18use client::{Output, OutputData, OutputMeta};
19use common_catalog::format_full_table_name;
20use common_datasource::file_format::Format;
21use common_datasource::lister::{Lister, Source};
22use common_datasource::object_store::{LocalFileAccess, build_backend, build_backend_for_write};
23use common_telemetry::{debug, error, info, tracing};
24use futures::future::try_join_all;
25use object_store::Entry;
26use regex::Regex;
27use session::context::QueryContextRef;
28use snafu::ResultExt;
29use table::requests::{CopyDatabaseRequest, CopyDirection, CopyTableRequest};
30use tokio::sync::Semaphore;
31
32use crate::error;
33use crate::statement::StatementExecutor;
34use crate::statement::database_copy::{
35    DatabaseExportFile, database_import_source, parse_parallelism_from_option_map,
36    validate_database_directory,
37};
38
39pub(crate) const COPY_DATABASE_TIME_START_KEY: &str = "start_time";
40pub(crate) const COPY_DATABASE_TIME_END_KEY: &str = "end_time";
41pub(crate) const CONTINUE_ON_ERROR_KEY: &str = "continue_on_error";
42
43impl StatementExecutor {
44    #[tracing::instrument(skip_all)]
45    pub(crate) async fn copy_database_to(
46        &self,
47        req: CopyDatabaseRequest,
48        ctx: QueryContextRef,
49    ) -> error::Result<Output> {
50        validate_database_directory(&req.location)?;
51        build_backend_for_write(&req.location, &req.connection, &self.local_file_access)
52            .await
53            .context(error::BuildBackendSnafu)?;
54
55        let parallelism = parse_parallelism_from_option_map(&req.with);
56        info!(
57            "Copy database {}.{} to dir: {}, time: {:?}, parallelism: {}",
58            req.catalog_name, req.schema_name, req.location, req.time_range, parallelism
59        );
60        let tables = self
61            .capture_database_export_tables(&req, None, &ctx)
62            .await?;
63        let num_tables = tables.len();
64
65        let suffix = Format::try_from(&req.with)
66            .context(error::ParseFileFormatSnafu)?
67            .suffix();
68
69        let mut tasks = Vec::with_capacity(num_tables);
70        let semaphore = Arc::new(Semaphore::new(parallelism));
71
72        for (i, table) in tables.into_iter().enumerate() {
73            let table_name = table.table_info().name.clone();
74            let semaphore_moved = semaphore.clone();
75            let table_file = DatabaseExportFile::new(&req.location, &table_name, suffix)?.location;
76            let table_no = i + 1;
77            let moved_ctx = ctx.clone();
78            let full_table_name =
79                format_full_table_name(&req.catalog_name, &req.schema_name, &table_name);
80            let copy_table_req = CopyTableRequest {
81                catalog_name: req.catalog_name.clone(),
82                schema_name: req.schema_name.clone(),
83                table_name,
84                location: table_file.clone(),
85                with: req.with.clone(),
86                connection: req.connection.clone(),
87                pattern: None,
88                direction: CopyDirection::Export,
89                timestamp_range: req.time_range,
90                limit: None,
91            };
92
93            tasks.push(async move {
94                let _permit = semaphore_moved.acquire().await.unwrap();
95                info!(
96                    "Copy table({}/{}): {} to {}",
97                    table_no, num_tables, full_table_name, table_file
98                );
99                self.copy_captured_table_to(table, copy_table_req, moved_ctx)
100                    .await
101            });
102        }
103
104        let results = try_join_all(tasks).await?;
105        let exported_rows = results.into_iter().sum();
106        Ok(Output::new_with_affected_rows(exported_rows))
107    }
108
109    /// Imports data to database from a given location and returns total rows imported.
110    #[tracing::instrument(skip_all)]
111    pub(crate) async fn copy_database_from(
112        &self,
113        req: CopyDatabaseRequest,
114        ctx: QueryContextRef,
115    ) -> error::Result<Output> {
116        validate_database_directory(&req.location)?;
117
118        let parallelism = parse_parallelism_from_option_map(&req.with);
119        info!(
120            "Copy database {}.{} from dir: {}, time: {:?}, parallelism: {}",
121            req.catalog_name, req.schema_name, req.location, req.time_range, parallelism
122        );
123        let suffix = Format::try_from(&req.with)
124            .context(error::ParseFileFormatSnafu)?
125            .suffix();
126
127        let entries = list_files_to_copy(&req, suffix, &self.local_file_access).await?;
128
129        let continue_on_error = req
130            .with
131            .get(CONTINUE_ON_ERROR_KEY)
132            .and_then(|v| bool::from_str(v).ok())
133            .unwrap_or(false);
134
135        let mut tasks = Vec::with_capacity(entries.len());
136        let semaphore = Arc::new(Semaphore::new(parallelism));
137
138        for e in entries {
139            let (table_name, location) = match database_import_source(&req.location, e.path()) {
140                Ok(source) => source,
141                Err(err) => {
142                    if continue_on_error {
143                        error!(err; "Failed to import table from file: {:?}", e);
144                        continue;
145                    } else {
146                        return Err(err);
147                    }
148                }
149            };
150
151            let req = CopyTableRequest {
152                catalog_name: req.catalog_name.clone(),
153                schema_name: req.schema_name.clone(),
154                table_name: table_name.clone(),
155                location,
156                with: req.with.clone(),
157                connection: req.connection.clone(),
158                pattern: None,
159                direction: CopyDirection::Import,
160                timestamp_range: None,
161                limit: None,
162            };
163            let moved_ctx = ctx.clone();
164            let moved_table_name = table_name.clone();
165            let moved_semaphore = semaphore.clone();
166            tasks.push(async move {
167                let _permit = moved_semaphore.acquire().await.unwrap();
168                debug!("Copy table, arg: {:?}", req);
169                match self.copy_table_from(req, moved_ctx).await {
170                    Ok(o) => {
171                        let (rows, cost) = o.extract_rows_and_cost();
172                        Ok((rows, cost))
173                    }
174                    Err(err) => {
175                        if continue_on_error {
176                            error!(err; "Failed to import file to table: {}", moved_table_name);
177                            Ok((0, 0))
178                        } else {
179                            Err(err)
180                        }
181                    }
182                }
183            });
184        }
185
186        let results = try_join_all(tasks).await?;
187        let (rows_inserted, insert_cost) = results
188            .into_iter()
189            .fold((0, 0), |(acc_rows, acc_cost), (rows, cost)| {
190                (acc_rows + rows, acc_cost + cost)
191            });
192
193        Ok(Output::new(
194            OutputData::AffectedRows(rows_inserted),
195            OutputMeta::new_with_cost(insert_cost),
196        ))
197    }
198}
199
200/// Lists all files with expected suffix that can be imported to database.
201async fn list_files_to_copy(
202    req: &CopyDatabaseRequest,
203    suffix: &str,
204    local_file_access: &LocalFileAccess,
205) -> error::Result<Vec<Entry>> {
206    let object_store = build_backend(&req.location, &req.connection, local_file_access)
207        .await
208        .context(error::BuildBackendSnafu)?;
209
210    let pattern = Regex::try_from(format!(".*{}", suffix)).context(error::BuildRegexSnafu)?;
211    let lister = Lister::new(
212        object_store.clone(),
213        Source::Dir,
214        "/".to_string(),
215        Some(pattern),
216    );
217    lister.list().await.context(error::ListObjectsSnafu)
218}
219
220#[cfg(test)]
221mod tests {
222    use std::collections::HashSet;
223
224    use common_datasource::object_store::LocalFileAccess;
225    use object_store::ObjectStore;
226    use object_store::services::Fs;
227    use object_store::util::normalize_dir;
228    #[cfg(not(windows))]
229    use path_slash::PathExt;
230    use table::requests::CopyDatabaseRequest;
231
232    use crate::statement::copy_database::list_files_to_copy;
233    use crate::statement::database_copy::database_import_source;
234
235    #[tokio::test]
236    async fn test_list_files_and_parse_table_name() {
237        let dir = common_test_util::temp_dir::create_temp_dir("test_list_files_to_copy");
238        let store_dir = normalize_dir(dir.path().to_str().unwrap());
239        let builder = Fs::default().root(&store_dir);
240        let object_store = ObjectStore::new(builder).unwrap();
241        object_store.write("a.parquet", "").await.unwrap();
242        object_store.write("b.parquet", "").await.unwrap();
243        object_store.write("c.csv", "").await.unwrap();
244        object_store.write("d", "").await.unwrap();
245        object_store.write("e.f.parquet", "").await.unwrap();
246
247        #[cfg(not(windows))]
248        let location = normalize_dir(&dir.path().to_slash().unwrap());
249        #[cfg(windows)]
250        let location = format!("{}\\", dir.path().display());
251        let request = CopyDatabaseRequest {
252            catalog_name: "catalog_0".to_string(),
253            schema_name: "schema_0".to_string(),
254            location,
255            with: [("FORMAT".to_string(), "parquet".to_string())]
256                .into_iter()
257                .collect(),
258            connection: Default::default(),
259            time_range: None,
260        };
261        let local_file_access = LocalFileAccess::sandboxed(dir.path()).unwrap();
262        let listed = list_files_to_copy(&request, ".parquet", &local_file_access)
263            .await
264            .unwrap()
265            .into_iter()
266            .map(|e| {
267                database_import_source(&request.location, e.path())
268                    .unwrap()
269                    .0
270            })
271            .collect::<HashSet<_>>();
272
273        assert_eq!(
274            ["a".to_string(), "b".to_string(), "e.f".to_string()]
275                .into_iter()
276                .collect::<HashSet<_>>(),
277            listed
278        );
279    }
280}