Skip to main content

operator/statement/
export_database.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Request-scoped database export preparation. The frontend authorizes the entire
16//! captured selection before preparation; these trusted methods do not check ACLs.
17
18use std::collections::{BTreeMap, HashSet};
19use std::future::Future;
20
21use common_datasource::file_format::Format;
22use common_datasource::object_store::{FILE_SCHEMA, FS_SCHEMA, build_backend_for_write, parse_url};
23use common_meta::key::table_route::TableRouteValue;
24use futures::StreamExt;
25use futures::stream::FuturesUnordered;
26use session::context::QueryContextRef;
27use snafu::{OptionExt, ResultExt, ensure};
28use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME};
29use table::TableRef;
30use table::metadata::TableType;
31use table::requests::{CopyDatabaseRequest, CopyDirection, CopyTableRequest};
32use tokio_util::sync::CancellationToken;
33
34use crate::error::{self, InvalidDatabaseExportSnafu, Result};
35use crate::statement::StatementExecutor;
36use crate::statement::database_copy::{
37    DatabaseExportFile, parse_parallelism_from_option_map, validate_database_directory,
38};
39use crate::statement::export_logical_tables::{LogicalTableExport, LogicalTableExportLimits};
40
41/// A validated request-scoped selection, not a metadata snapshot or an ACL token.
42pub struct PreparedDatabaseExport {
43    request: CopyDatabaseRequest,
44    jobs: Vec<DatabaseExportJob>,
45}
46
47impl PreparedDatabaseExport {
48    /// Inspect physical grouping through the integration testing adapter.
49    #[cfg(feature = "testing")]
50    pub fn job_count_for_test(&self) -> usize {
51        self.jobs.len()
52    }
53}
54
55enum DatabaseExportJob {
56    Ordinary {
57        table: TableRef,
58        output: DatabaseExportFile,
59    },
60    Metric(LogicalTableExport),
61}
62
63impl StatementExecutor {
64    /// Validate all jobs and destinations before executing any query or writer.
65    /// Callers must authorize every captured table before calling this method.
66    pub async fn prepare_database_export(
67        &self,
68        req: CopyDatabaseRequest,
69        tables: Vec<TableRef>,
70    ) -> Result<PreparedDatabaseExport> {
71        validate_database_directory(&req.location)?;
72        let format = Format::try_from(&req.with).context(error::ParseFileFormatSnafu)?;
73        ensure!(
74            matches!(format, Format::Parquet(_)),
75            error::UnsupportedFormatSnafu { format }
76        );
77        let (scheme, _, _) = parse_url(&req.location).context(error::BuildBackendSnafu)?;
78        let local =
79            scheme.eq_ignore_ascii_case(FS_SCHEMA) || scheme.eq_ignore_ascii_case(FILE_SCHEMA);
80        let mut filenames = HashSet::new();
81        let mut logical = Vec::new();
82        let mut jobs = Vec::new();
83        for table in tables {
84            let info = table.table_info();
85            let name = &info.name;
86            let output = DatabaseExportFile::new(&req.location, name, ".parquet")?;
87            ensure!(
88                filenames.insert(if local {
89                    output.path.to_ascii_lowercase()
90                } else {
91                    output.path.clone()
92                }),
93                InvalidDatabaseExportSnafu {
94                    reason: format!("duplicate output name: {name}")
95                }
96            );
97            ensure!(
98                info.catalog_name == req.catalog_name
99                    && info.schema_name == req.schema_name
100                    && table.table_type() == TableType::Base,
101                InvalidDatabaseExportSnafu {
102                    reason: "expected base tables in the selected schema"
103                }
104            );
105            if info.meta.engine == METRIC_ENGINE_NAME {
106                ensure!(
107                    info.meta
108                        .options
109                        .extra_options
110                        .contains_key(LOGICAL_TABLE_METADATA_KEY),
111                    InvalidDatabaseExportSnafu {
112                        reason: "expected a Metric logical table"
113                    }
114                );
115                logical.push(table);
116            } else {
117                jobs.push(DatabaseExportJob::Ordinary { table, output });
118            }
119        }
120        let ids = logical
121            .iter()
122            .map(|t| t.table_info().table_id())
123            .collect::<Vec<_>>();
124        let routes = self
125            .table_metadata_manager
126            .table_route_manager()
127            .table_route_storage()
128            .batch_get(&ids)
129            .await
130            .context(error::TableMetadataManagerSnafu)?;
131        let mut groups = BTreeMap::<_, Vec<TableRef>>::new();
132        for (table, route) in logical.into_iter().zip(routes) {
133            let Some(TableRouteValue::Logical(route)) = route else {
134                return InvalidDatabaseExportSnafu {
135                    reason: format!(
136                        "missing or non-logical route for {}",
137                        table.table_info().table_id()
138                    ),
139                }
140                .fail();
141            };
142            groups
143                .entry(route.physical_table_id())
144                .or_default()
145                .push(table);
146        }
147        let physical_ids = groups.keys().copied().collect::<Vec<_>>();
148        let mut physical = self
149            .catalog_manager
150            .tables_by_ids(&req.catalog_name, &req.schema_name, &physical_ids)
151            .await
152            .context(error::CatalogSnafu)?
153            .into_iter()
154            .map(|t| (t.table_info().table_id(), t))
155            .collect::<BTreeMap<_, _>>();
156        for (id, tables) in groups {
157            let table = physical
158                .remove(&id)
159                .with_context(|| InvalidDatabaseExportSnafu {
160                    reason: format!("missing physical table {id} in the selected schema"),
161                })?;
162            jobs.push(DatabaseExportJob::Metric(
163                LogicalTableExport::try_new_in_directory(table, &tables, &req.location)?,
164            ));
165        }
166        build_backend_for_write(&req.location, &req.connection, &self.local_file_access)
167            .await
168            .context(error::BuildBackendSnafu)?;
169        Ok(PreparedDatabaseExport { request: req, jobs })
170    }
171}
172
173/// Returned only after every started job has closed or drained its owned I/O.
174#[derive(Debug)]
175pub struct DatabaseExportSummary {
176    pub rows: usize,
177    pub output_files: Vec<String>,
178}
179
180impl StatementExecutor {
181    /// Execute with one job budget. The caller must keep this future alive until
182    /// it returns, including after cooperative cancellation. Closed files remain
183    /// owned by the caller's attempt; this method does not publish completion.
184    pub async fn export_database(
185        &self,
186        plan: PreparedDatabaseExport,
187        cancellation: &CancellationToken,
188        ctx: QueryContextRef,
189    ) -> Result<DatabaseExportSummary> {
190        let mut output_files = Vec::new();
191        for job in &plan.jobs {
192            match job {
193                DatabaseExportJob::Ordinary { output, .. } => {
194                    output_files.push(output.location.clone())
195                }
196                DatabaseExportJob::Metric(unit) => {
197                    output_files.extend(unit.output_files().map(|file| file.location.clone()))
198                }
199            }
200        }
201        output_files.sort();
202        let req = &plan.request;
203        let rows = run_database_export_jobs(
204            plan.jobs,
205            parse_parallelism_from_option_map(&req.with),
206            cancellation,
207            |job, token| {
208                let ctx = ctx.clone();
209                async move {
210                    match job {
211                        DatabaseExportJob::Metric(unit) => self
212                            .export_logical_tables(
213                                &unit,
214                                &req.location,
215                                &req.connection,
216                                req.time_range.as_ref(),
217                                LogicalTableExportLimits::default(),
218                                &token,
219                                ctx,
220                            )
221                            .await
222                            .map(|summary| summary.rows),
223                        DatabaseExportJob::Ordinary { table, output } => {
224                            let info = table.table_info();
225                            let copy = CopyTableRequest {
226                                catalog_name: info.catalog_name.clone(),
227                                schema_name: info.schema_name.clone(),
228                                table_name: info.name.clone(),
229                                location: output.location,
230                                with: req.with.clone(),
231                                connection: req.connection.clone(),
232                                pattern: None,
233                                direction: CopyDirection::Export,
234                                timestamp_range: req.time_range,
235                                limit: None,
236                            };
237                            self.copy_captured_table_to(table, copy, ctx).await
238                        }
239                    }
240                }
241            },
242        )
243        .await?;
244        Ok(DatabaseExportSummary { rows, output_files })
245    }
246}
247
248async fn run_database_export_jobs<J, F: Future<Output = Result<usize>>>(
249    jobs: impl IntoIterator<Item = J>,
250    parallelism: usize,
251    cancellation: &CancellationToken,
252    mut run: impl FnMut(J, CancellationToken) -> F,
253) -> Result<usize> {
254    let token = CancellationToken::new();
255    let mut jobs = jobs.into_iter();
256    let mut active = FuturesUnordered::new();
257    let mut first_error = None;
258    let mut rows = 0;
259    loop {
260        while first_error.is_none()
261            && !cancellation.is_cancelled()
262            && active.len() < parallelism.max(1)
263        {
264            let Some(job) = jobs.next() else { break };
265            active.push(run(job, token.clone()));
266        }
267        if first_error.is_none() && cancellation.is_cancelled() {
268            first_error = Some(error::DatabaseExportCancelledSnafu.build());
269            token.cancel();
270        }
271        if active.is_empty() {
272            break;
273        }
274        let result = tokio::select! {
275            biased;
276            _ = cancellation.cancelled(), if first_error.is_none() => {
277                first_error = Some(error::DatabaseExportCancelledSnafu.build());
278                token.cancel();
279                continue;
280            }
281            result = active.next() => result,
282        };
283        match result {
284            Some(Ok(count)) => rows += count,
285            Some(Err(err)) if first_error.is_none() => {
286                first_error = Some(err);
287                token.cancel();
288            }
289            Some(Err(err)) => common_telemetry::warn!(err; "Failed to drain database export job"),
290            None => break,
291        }
292    }
293    match first_error {
294        Some(err) => Err(err),
295        None => Ok(rows),
296    }
297}
298
299#[cfg(test)]
300mod tests {
301    use std::sync::Arc;
302    use std::sync::atomic::{AtomicUsize, Ordering};
303
304    use tokio::sync::{Semaphore, mpsc};
305
306    use super::*;
307
308    #[tokio::test]
309    async fn bounded_admission_and_drain() {
310        for cancel in [false, true] {
311            let cancellation = CancellationToken::new();
312            let start_error = Arc::new(Semaphore::new(0));
313            let finish_io = Arc::new(Semaphore::new(0));
314            let (started, mut receiver) = mpsc::unbounded_channel();
315            let finished = Arc::new(AtomicUsize::new(0));
316            let task = tokio::spawn({
317                let cancellation = cancellation.clone();
318                let start_error = start_error.clone();
319                let finish_io = finish_io.clone();
320                let finished = finished.clone();
321                async move {
322                    run_database_export_jobs(0..4, 2, &cancellation, |job, token| {
323                        let started = started.clone();
324                        let start_error = start_error.clone();
325                        let finish_io = finish_io.clone();
326                        let finished = finished.clone();
327                        async move {
328                            started.send(job).unwrap();
329                            if job == 0 {
330                                if cancel {
331                                    token.cancelled().await;
332                                } else {
333                                    start_error.acquire().await.unwrap().forget();
334                                }
335                                return InvalidDatabaseExportSnafu {
336                                    reason: "first error",
337                                }
338                                .fail();
339                            }
340                            // Ordinary COPY continues its I/O even when Metric jobs cancel.
341                            token.cancelled().await;
342                            started.send(10).unwrap();
343                            finish_io.acquire().await.unwrap().forget();
344                            finished.fetch_add(1, Ordering::SeqCst);
345                            InvalidDatabaseExportSnafu {
346                                reason: "drain error",
347                            }
348                            .fail()
349                        }
350                    })
351                    .await
352                }
353            });
354            assert_eq!(receiver.recv().await, Some(0));
355            assert_eq!(receiver.recv().await, Some(1));
356            assert!(receiver.try_recv().is_err());
357            if cancel {
358                cancellation.cancel();
359            } else {
360                start_error.add_permits(1);
361            }
362            assert_eq!(receiver.recv().await, Some(10));
363            assert!(!task.is_finished());
364            finish_io.add_permits(1);
365            let err = task.await.unwrap().unwrap_err();
366            if cancel {
367                assert!(matches!(err, error::Error::DatabaseExportCancelled { .. }));
368            } else {
369                assert!(
370                    matches!(err, error::Error::InvalidDatabaseExport { reason } if reason == "first error")
371                );
372            }
373            assert_eq!(finished.load(Ordering::SeqCst), 1);
374            assert_eq!(receiver.recv().await, None);
375        }
376    }
377
378    #[tokio::test]
379    async fn cancellation_before_admission_and_successful_refill() {
380        let token = CancellationToken::new();
381        token.cancel();
382        let result = run_database_export_jobs(0..4, 2, &token, |_, _| async {
383            panic!("cancelled job admitted")
384        })
385        .await;
386        assert!(matches!(
387            result,
388            Err(error::Error::DatabaseExportCancelled { .. })
389        ));
390        let started = AtomicUsize::new(0);
391        let result = run_database_export_jobs(0..7, 2, &CancellationToken::new(), |job, _| {
392            started.fetch_add(1, Ordering::SeqCst);
393            async move { Ok(job) }
394        })
395        .await
396        .unwrap();
397        assert_eq!(result, 21);
398        assert_eq!(started.load(Ordering::SeqCst), 7);
399    }
400}