Skip to main content

operator/statement/
copy_table_from.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeSet, HashMap, HashSet};
16use std::future::Future;
17use std::pin::Pin;
18use std::sync::Arc;
19use std::task::{Context, Poll};
20
21use client::{Output, OutputData, OutputMeta};
22use common_base::readable_size::ReadableSize;
23use common_datasource::file_format::csv::{
24    CsvFormat, is_skippable_arrow_error, tolerant_csv_stream,
25};
26use common_datasource::file_format::json::JsonFormat;
27use common_datasource::file_format::orc::{ReaderAdapter, infer_orc_schema, new_orc_stream_reader};
28use common_datasource::file_format::{FileFormat, Format, file_to_stream};
29use common_datasource::lister::{Lister, Source};
30use common_datasource::object_store::build_backend_with_path;
31use common_query::{OutputCost, OutputRows};
32use common_recordbatch::DfSendableRecordBatchStream;
33use common_recordbatch::adapter::RecordBatchStreamTypeAdapter;
34use common_telemetry::{debug, tracing};
35use datafusion::datasource::physical_plan::{CsvSource, FileSource, JsonSource};
36use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
37use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata;
38use datafusion_common::DataFusionError;
39use datafusion_common::arrow::error::ArrowError;
40use datafusion_common::config::CsvOptions;
41use datafusion_expr::Expr;
42use datatypes::arrow::compute::can_cast_types;
43use datatypes::arrow::datatypes::{DataType as ArrowDataType, Schema, SchemaRef};
44use datatypes::arrow::record_batch::RecordBatch;
45use datatypes::vectors::Helper;
46use futures_util::StreamExt;
47use object_store::{Entry, EntryMode, ObjectStore};
48use regex::Regex;
49use session::context::QueryContextRef;
50use snafu::{ResultExt, ensure};
51use table::requests::{CopyTableRequest, InsertRequest};
52use table::table_reference::TableReference;
53use tokio_util::compat::FuturesAsyncReadCompatExt;
54
55use crate::error::{self, IntoVectorsSnafu, Result};
56use crate::statement::StatementExecutor;
57
58const DEFAULT_BATCH_SIZE: usize = 8192;
59const DEFAULT_READ_BUFFER: usize = 256 * 1024;
60
61enum FileMetadata {
62    Parquet {
63        schema: SchemaRef,
64        metadata: ArrowReaderMetadata,
65        path: String,
66    },
67    Orc {
68        schema: SchemaRef,
69        path: String,
70    },
71    Json {
72        schema: SchemaRef,
73        format: JsonFormat,
74        path: String,
75    },
76    Csv {
77        schema: SchemaRef,
78        format: CsvFormat,
79        path: String,
80    },
81}
82
83impl FileMetadata {
84    /// Returns the [SchemaRef]
85    pub fn schema(&self) -> &SchemaRef {
86        match self {
87            FileMetadata::Parquet { schema, .. } => schema,
88            FileMetadata::Orc { schema, .. } => schema,
89            FileMetadata::Json { schema, .. } => schema,
90            FileMetadata::Csv { schema, .. } => schema,
91        }
92    }
93}
94
95impl StatementExecutor {
96    async fn list_copy_from_entries(
97        &self,
98        req: &CopyTableRequest,
99    ) -> Result<(ObjectStore, Vec<Entry>)> {
100        let backend =
101            build_backend_with_path(&req.location, &req.connection, &self.local_file_access)
102                .await
103                .context(error::BuildBackendSnafu)?;
104        let regex = req
105            .pattern
106            .as_ref()
107            .map(|x| Regex::new(x))
108            .transpose()
109            .context(error::BuildRegexSnafu)?;
110
111        let source = if let Some(filename) = backend.object_path {
112            Source::Filename(filename)
113        } else {
114            Source::Dir
115        };
116
117        let lister = Lister::new(
118            backend.object_store.clone(),
119            source.clone(),
120            req.location.clone(),
121            regex,
122        );
123
124        let entries = lister.list().await.context(error::ListObjectsSnafu)?;
125        debug!(
126            "Copy from location: {:?}, {source:?}, entries: {entries:?}",
127            req.location
128        );
129        Ok((backend.object_store, entries))
130    }
131
132    async fn collect_metadata(
133        &self,
134        object_store: &ObjectStore,
135        format: Format,
136        path: String,
137    ) -> Result<FileMetadata> {
138        match format {
139            Format::Csv(format) => Ok(FileMetadata::Csv {
140                schema: Arc::new(
141                    format
142                        .infer_schema(object_store, &path)
143                        .await
144                        .context(error::InferSchemaSnafu { path: &path })?,
145                ),
146                format,
147                path,
148            }),
149            Format::Json(format) => Ok(FileMetadata::Json {
150                schema: Arc::new(
151                    format
152                        .infer_schema(object_store, &path)
153                        .await
154                        .context(error::InferSchemaSnafu { path: &path })?,
155                ),
156                format,
157                path,
158            }),
159            Format::Parquet(_) => {
160                let meta = object_store
161                    .stat(&path)
162                    .await
163                    .context(error::ReadObjectSnafu { path: &path })?;
164                let mut reader = object_store
165                    .reader(&path)
166                    .await
167                    .context(error::ReadObjectSnafu { path: &path })?
168                    .into_futures_async_read(0..meta.content_length())
169                    .await
170                    .context(error::ReadObjectSnafu { path: &path })?
171                    .compat();
172                let metadata = ArrowReaderMetadata::load_async(&mut reader, Default::default())
173                    .await
174                    .context(error::ReadParquetMetadataSnafu)?;
175
176                Ok(FileMetadata::Parquet {
177                    schema: metadata.schema().clone(),
178                    metadata,
179                    path,
180                })
181            }
182            Format::Orc(_) => {
183                let meta = object_store
184                    .stat(&path)
185                    .await
186                    .context(error::ReadObjectSnafu { path: &path })?;
187
188                let reader = object_store
189                    .reader(&path)
190                    .await
191                    .context(error::ReadObjectSnafu { path: &path })?;
192
193                let schema = infer_orc_schema(ReaderAdapter::new(reader, meta.content_length()))
194                    .await
195                    .context(error::ReadOrcSnafu)?;
196
197                Ok(FileMetadata::Orc {
198                    schema: Arc::new(schema),
199                    path,
200                })
201            }
202        }
203    }
204
205    async fn build_read_stream(
206        &self,
207        compat_schema: SchemaRef,
208        object_store: &ObjectStore,
209        file_metadata: &FileMetadata,
210        projection: Vec<usize>,
211        filters: Vec<Expr>,
212    ) -> Result<DfSendableRecordBatchStream> {
213        match file_metadata {
214            FileMetadata::Csv {
215                format,
216                path,
217                schema,
218            } => {
219                let output_schema = Arc::new(
220                    compat_schema
221                        .project(&projection)
222                        .context(error::ProjectSchemaSnafu)?,
223                );
224
225                let options = CsvOptions::default()
226                    .with_has_header(format.has_header)
227                    .with_delimiter(format.delimiter);
228                let csv_source = CsvSource::new(schema.clone())
229                    .with_csv_options(options)
230                    .with_batch_size(DEFAULT_BATCH_SIZE);
231                let stream = if format.skip_bad_records {
232                    let reader_schema =
233                        csv_reader_schema_for_skip_bad_records(schema, &compat_schema);
234                    tolerant_csv_stream(
235                        object_store,
236                        path,
237                        Arc::new(reader_schema),
238                        projection.clone(),
239                        format,
240                    )
241                    .await
242                    .context(error::BuildFileStreamSnafu)?
243                } else {
244                    file_to_stream(
245                        object_store,
246                        path,
247                        csv_source,
248                        Some(projection),
249                        format.compression_type,
250                    )
251                    .await
252                    .context(error::BuildFileStreamSnafu)?
253                };
254
255                let stream = Box::pin(
256                    // The projection is already applied in the CSV reader when we created the stream,
257                    // so we pass None here to avoid double projection which would cause schema mismatch errors.
258                    RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
259                        .with_filter(filters)
260                        .context(error::PhysicalExprSnafu)?,
261                );
262                if format.skip_bad_records {
263                    Ok(Box::pin(SkipBadRecordsStream::new(stream, path)))
264                } else {
265                    Ok(stream)
266                }
267            }
268            FileMetadata::Json {
269                path,
270                format,
271                schema,
272            } => {
273                let output_schema = Arc::new(
274                    compat_schema
275                        .project(&projection)
276                        .context(error::ProjectSchemaSnafu)?,
277                );
278
279                let json_source =
280                    JsonSource::new(schema.clone()).with_batch_size(DEFAULT_BATCH_SIZE);
281                let stream = file_to_stream(
282                    object_store,
283                    path,
284                    json_source,
285                    Some(projection),
286                    format.compression_type,
287                )
288                .await
289                .context(error::BuildFileStreamSnafu)?;
290
291                Ok(Box::pin(
292                    // The projection is already applied in the JSON reader when we created the stream,
293                    // so we pass None here to avoid double projection which would cause schema mismatch errors.
294                    RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
295                        .with_filter(filters)
296                        .context(error::PhysicalExprSnafu)?,
297                ))
298            }
299            FileMetadata::Parquet { metadata, path, .. } => {
300                let meta = object_store
301                    .stat(path)
302                    .await
303                    .context(error::ReadObjectSnafu { path })?;
304                let reader = object_store
305                    .reader_with(path)
306                    .chunk(DEFAULT_READ_BUFFER)
307                    .await
308                    .context(error::ReadObjectSnafu { path })?
309                    .into_futures_async_read(0..meta.content_length())
310                    .await
311                    .context(error::ReadObjectSnafu { path })?
312                    .compat();
313                let builder =
314                    ParquetRecordBatchStreamBuilder::new_with_metadata(reader, metadata.clone());
315                let stream = builder
316                    .build()
317                    .context(error::BuildParquetRecordBatchStreamSnafu)?;
318
319                let output_schema = Arc::new(
320                    compat_schema
321                        .project(&projection)
322                        .context(error::ProjectSchemaSnafu)?,
323                );
324                Ok(Box::pin(
325                    RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
326                        .with_filter(filters)
327                        .context(error::PhysicalExprSnafu)?,
328                ))
329            }
330            FileMetadata::Orc { path, .. } => {
331                let meta = object_store
332                    .stat(path)
333                    .await
334                    .context(error::ReadObjectSnafu { path })?;
335
336                let reader = object_store
337                    .reader_with(path)
338                    .chunk(DEFAULT_READ_BUFFER)
339                    .await
340                    .context(error::ReadObjectSnafu { path })?;
341                let stream =
342                    new_orc_stream_reader(ReaderAdapter::new(reader, meta.content_length()))
343                        .await
344                        .context(error::ReadOrcSnafu)?;
345
346                let output_schema = Arc::new(
347                    compat_schema
348                        .project(&projection)
349                        .context(error::ProjectSchemaSnafu)?,
350                );
351
352                Ok(Box::pin(
353                    RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
354                        .with_filter(filters)
355                        .context(error::PhysicalExprSnafu)?,
356                ))
357            }
358        }
359    }
360
361    #[tracing::instrument(skip_all)]
362    pub async fn copy_table_from(
363        &self,
364        req: CopyTableRequest,
365        query_ctx: QueryContextRef,
366    ) -> Result<Output> {
367        let table_ref = TableReference {
368            catalog: &req.catalog_name,
369            schema: &req.schema_name,
370            table: &req.table_name,
371        };
372        let table = self.get_table(&table_ref).await?;
373        let format = Format::try_from(&req.with).context(error::ParseFileFormatSnafu)?;
374        let (object_store, entries) = self.list_copy_from_entries(&req).await?;
375        let mut files = Vec::with_capacity(entries.len());
376        let table_schema = table.schema().arrow_schema().clone();
377        let filters = table
378            .schema()
379            .timestamp_column()
380            .and_then(|c| {
381                common_query::logical_plan::build_same_type_ts_filter(c, req.timestamp_range)
382            })
383            .into_iter()
384            .collect::<Vec<_>>();
385
386        for entry in entries.iter() {
387            if entry.metadata().mode() != EntryMode::FILE {
388                continue;
389            }
390            let path = entry.path();
391            let file_metadata = self
392                .collect_metadata(&object_store, format.clone(), path.to_string())
393                .await?;
394
395            validate_csv_headers_if_required(&file_metadata, &table_schema)?;
396            let schema_mapping = copy_from_schema_mapping(&file_metadata, &table_schema);
397            let projected_file_schema = Arc::new(
398                file_metadata
399                    .schema()
400                    .project(&schema_mapping.file_projection)
401                    .context(error::ProjectSchemaSnafu)?,
402            );
403            let projected_table_schema = Arc::new(
404                table_schema
405                    .project(&schema_mapping.table_projection)
406                    .context(error::ProjectSchemaSnafu)?,
407            );
408            ensure_schema_compatible(&projected_file_schema, &projected_table_schema)?;
409
410            files.push((
411                Arc::new(schema_mapping.compat_file_schema),
412                schema_mapping.file_projection,
413                projected_table_schema,
414                file_metadata,
415            ))
416        }
417
418        let mut rows_inserted = 0;
419        let mut insert_cost = 0;
420        let max_insert_rows = req
421            .limit
422            .map(|n| {
423                usize::try_from(n).map_err(|_| {
424                    error::InvalidCopyParameterSnafu {
425                        key: "limit".to_string(),
426                        value: n.to_string(),
427                    }
428                    .build()
429                })
430            })
431            .transpose()?;
432        if max_insert_rows == Some(0) {
433            return Ok(gen_insert_output(rows_inserted, insert_cost));
434        }
435
436        let mut accepted_rows = 0;
437        for (compat_schema, file_schema_projection, projected_table_schema, file_metadata) in files
438        {
439            let mut stream = self
440                .build_read_stream(
441                    compat_schema,
442                    &object_store,
443                    &file_metadata,
444                    file_schema_projection,
445                    filters.clone(),
446                )
447                .await?;
448
449            let fields = projected_table_schema
450                .fields()
451                .iter()
452                .map(|f| f.name().clone())
453                .collect::<Vec<_>>();
454
455            // TODO(hl): make this configurable through options.
456            let pending_mem_threshold = ReadableSize::mb(32).as_bytes();
457            let mut pending_mem_size = 0;
458            let mut pending = vec![];
459
460            while let Some(r) = stream.next().await {
461                let record_batch = r.context(error::ReadDfRecordBatchSnafu)?;
462                let record_batch = if let Some(max_insert_rows) = max_insert_rows {
463                    let remaining_rows = max_insert_rows - accepted_rows;
464                    if record_batch.num_rows() > remaining_rows {
465                        record_batch.slice(0, remaining_rows)
466                    } else {
467                        record_batch
468                    }
469                } else {
470                    record_batch
471                };
472                let record_batch_rows = record_batch.num_rows();
473                let vectors =
474                    Helper::try_into_vectors(record_batch.columns()).context(IntoVectorsSnafu)?;
475
476                pending_mem_size += vectors.iter().map(|v| v.memory_size()).sum::<usize>();
477
478                let columns_values = fields
479                    .iter()
480                    .cloned()
481                    .zip(vectors)
482                    .collect::<HashMap<_, _>>();
483
484                pending.push(self.inserter.handle_table_insert(
485                    InsertRequest {
486                        catalog_name: req.catalog_name.clone(),
487                        schema_name: req.schema_name.clone(),
488                        table_name: req.table_name.clone(),
489                        columns_values,
490                        skip_wal: query_ctx.skip_wal(),
491                    },
492                    query_ctx.clone(),
493                ));
494                accepted_rows += record_batch_rows;
495
496                if pending_mem_size as u64 >= pending_mem_threshold {
497                    let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
498                    rows_inserted += rows;
499                    insert_cost += cost;
500                }
501
502                if let Some(max_insert_rows) = max_insert_rows
503                    && accepted_rows == max_insert_rows
504                {
505                    if !pending.is_empty() {
506                        let (rows, cost) =
507                            batch_insert(&mut pending, &mut pending_mem_size).await?;
508                        rows_inserted += rows;
509                        insert_cost += cost;
510                    }
511                    return Ok(gen_insert_output(rows_inserted, insert_cost));
512                }
513            }
514
515            if !pending.is_empty() {
516                let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
517                rows_inserted += rows;
518                insert_cost += cost;
519            }
520        }
521
522        Ok(gen_insert_output(rows_inserted, insert_cost))
523    }
524}
525
526fn gen_insert_output(rows_inserted: usize, insert_cost: usize) -> Output {
527    Output::new(
528        OutputData::AffectedRows(rows_inserted),
529        OutputMeta::new_with_cost(insert_cost),
530    )
531}
532
533struct SkipBadRecordsStream {
534    inner: DfSendableRecordBatchStream,
535    path: String,
536}
537
538impl SkipBadRecordsStream {
539    fn new(inner: DfSendableRecordBatchStream, path: impl Into<String>) -> Self {
540        Self {
541            inner,
542            path: path.into(),
543        }
544    }
545}
546
547impl datafusion::physical_plan::RecordBatchStream for SkipBadRecordsStream {
548    fn schema(&self) -> SchemaRef {
549        self.inner.schema()
550    }
551}
552
553impl futures::Stream for SkipBadRecordsStream {
554    type Item = datafusion_common::Result<RecordBatch>;
555
556    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
557        let this = self.get_mut();
558        loop {
559            match this.inner.as_mut().poll_next(cx) {
560                Poll::Ready(Some(Err(error))) if is_skippable_record_error(&error) => {
561                    common_telemetry::warn!(
562                        "Skipping bad record while copying from {}: {}",
563                        this.path,
564                        error
565                    );
566                    continue;
567                }
568                other => return other,
569            }
570        }
571    }
572}
573
574fn is_skippable_record_error(error: &DataFusionError) -> bool {
575    match error {
576        DataFusionError::ArrowError(error, _) => is_skippable_arrow_error(error),
577        DataFusionError::External(error) => error
578            .downcast_ref::<ArrowError>()
579            .is_some_and(is_skippable_arrow_error),
580        DataFusionError::Context(_, error) => is_skippable_record_error(error),
581        _ => false,
582    }
583}
584
585/// Executes all pending inserts all at once, drain pending requests and reset pending bytes.
586async fn batch_insert(
587    pending: &mut Vec<impl Future<Output = Result<Output>>>,
588    pending_bytes: &mut usize,
589) -> Result<(OutputRows, OutputCost)> {
590    let batch = pending.drain(..);
591    let result = futures::future::try_join_all(batch)
592        .await?
593        .iter()
594        .map(|o| o.extract_rows_and_cost())
595        .reduce(|(a, b), (c, d)| (a + c, b + d))
596        .unwrap_or((0, 0));
597    *pending_bytes = 0;
598    Ok(result)
599}
600
601/// Custom type compatibility check for GreptimeDB that handles Map -> Binary (JSON) conversion
602fn can_cast_types_for_greptime(from: &ArrowDataType, to: &ArrowDataType) -> bool {
603    // Handle Map -> Binary conversion for JSON types
604    if let ArrowDataType::Map(_, _) = from
605        && let ArrowDataType::Binary = to
606    {
607        return true;
608    }
609
610    // For all other cases, use Arrow's built-in can_cast_types
611    can_cast_types(from, to)
612}
613
614fn csv_reader_schema_for_skip_bad_records(file: &SchemaRef, compat: &SchemaRef) -> Schema {
615    let fields = file
616        .fields()
617        .iter()
618        .enumerate()
619        .map(|(idx, file_field)| match compat.fields().get(idx) {
620            Some(compat_field) if can_csv_reader_parse_type(compat_field.data_type()) => {
621                compat_field.clone()
622            }
623            _ => file_field.clone(),
624        })
625        .collect::<Vec<_>>();
626
627    Schema::new_with_metadata(fields, file.metadata().clone())
628}
629
630fn can_csv_reader_parse_type(data_type: &ArrowDataType) -> bool {
631    match data_type {
632        ArrowDataType::Boolean
633        | ArrowDataType::Decimal32(_, _)
634        | ArrowDataType::Decimal64(_, _)
635        | ArrowDataType::Decimal128(_, _)
636        | ArrowDataType::Decimal256(_, _)
637        | ArrowDataType::Int8
638        | ArrowDataType::Int16
639        | ArrowDataType::Int32
640        | ArrowDataType::Int64
641        | ArrowDataType::UInt8
642        | ArrowDataType::UInt16
643        | ArrowDataType::UInt32
644        | ArrowDataType::UInt64
645        | ArrowDataType::Float32
646        | ArrowDataType::Float64
647        | ArrowDataType::Date32
648        | ArrowDataType::Date64
649        | ArrowDataType::Time32(_)
650        | ArrowDataType::Time64(_)
651        | ArrowDataType::Timestamp(_, _)
652        | ArrowDataType::Null
653        | ArrowDataType::Utf8
654        | ArrowDataType::Utf8View => true,
655        ArrowDataType::Dictionary(_, value_type) => value_type.as_ref() == &ArrowDataType::Utf8,
656        _ => false,
657    }
658}
659
660fn ensure_schema_compatible(from: &SchemaRef, to: &SchemaRef) -> Result<()> {
661    let not_match = from
662        .fields
663        .iter()
664        .zip(to.fields.iter())
665        .map(|(l, r)| (l.data_type(), r.data_type()))
666        .enumerate()
667        .find(|(_, (l, r))| !can_cast_types_for_greptime(l, r));
668
669    if let Some((index, _)) = not_match {
670        error::InvalidSchemaSnafu {
671            index,
672            table_schema: to.to_string(),
673            file_schema: from.to_string(),
674        }
675        .fail()
676    } else {
677        Ok(())
678    }
679}
680
681fn validate_csv_headers_if_required(file_metadata: &FileMetadata, table: &SchemaRef) -> Result<()> {
682    let FileMetadata::Csv {
683        schema,
684        format,
685        path,
686    } = file_metadata
687    else {
688        return Ok(());
689    };
690
691    if !format.strict_headers {
692        return Ok(());
693    }
694
695    let mut seen_file_columns = HashSet::with_capacity(schema.fields().len());
696    let duplicate_columns = schema
697        .fields()
698        .iter()
699        .filter_map(|field| {
700            if seen_file_columns.insert(field.name().clone()) {
701                None
702            } else {
703                Some(field.name().clone())
704            }
705        })
706        .collect::<BTreeSet<_>>()
707        .into_iter()
708        .collect::<Vec<_>>();
709    let file_columns = seen_file_columns.into_iter().collect::<BTreeSet<_>>();
710    let table_columns = table
711        .fields()
712        .iter()
713        .map(|field| field.name().clone())
714        .collect::<BTreeSet<_>>();
715    let unknown_columns = file_columns
716        .difference(&table_columns)
717        .cloned()
718        .collect::<Vec<_>>();
719    let missing_columns = table_columns
720        .difference(&file_columns)
721        .cloned()
722        .collect::<Vec<_>>();
723
724    ensure!(
725        unknown_columns.is_empty() && missing_columns.is_empty() && duplicate_columns.is_empty(),
726        error::CsvHeaderMismatchSnafu {
727            path,
728            unknown_columns,
729            missing_columns,
730            duplicate_columns,
731        }
732    );
733
734    Ok(())
735}
736
737/// Generates a maybe compatible schema of the file schema.
738///
739/// If there is a field is found in table schema,
740/// copy the field data type to maybe compatible schema(`compatible_fields`).
741fn generated_schema_projection_and_compatible_file_schema(
742    file: &SchemaRef,
743    table: &SchemaRef,
744) -> (Vec<usize>, Vec<usize>, Schema) {
745    let mut file_projection = Vec::with_capacity(file.fields.len());
746    let mut table_projection = Vec::with_capacity(file.fields.len());
747    let mut compatible_fields = file.fields.iter().cloned().collect::<Vec<_>>();
748    for (file_idx, file_field) in file.fields.iter().enumerate() {
749        if let Some((table_idx, table_field)) = table.fields.find(file_field.name()) {
750            file_projection.push(file_idx);
751            table_projection.push(table_idx);
752
753            // Safety: the compatible_fields has same length as file schema
754            compatible_fields[file_idx] = table_field.clone();
755        }
756    }
757
758    (
759        file_projection,
760        table_projection,
761        Schema::new(compatible_fields),
762    )
763}
764
765struct CopyFromSchemaMapping {
766    file_projection: Vec<usize>,
767    table_projection: Vec<usize>,
768    compat_file_schema: Schema,
769}
770
771fn copy_from_schema_mapping(
772    file_metadata: &FileMetadata,
773    table: &SchemaRef,
774) -> CopyFromSchemaMapping {
775    match file_metadata {
776        FileMetadata::Csv { schema, format, .. } if !format.has_header => {
777            generated_positional_schema_projection_and_compatible_file_schema(schema, table)
778        }
779        _ => {
780            let (file_projection, table_projection, compat_file_schema) =
781                generated_schema_projection_and_compatible_file_schema(
782                    file_metadata.schema(),
783                    table,
784                );
785            CopyFromSchemaMapping {
786                file_projection,
787                table_projection,
788                compat_file_schema,
789            }
790        }
791    }
792}
793
794fn generated_positional_schema_projection_and_compatible_file_schema(
795    file: &SchemaRef,
796    table: &SchemaRef,
797) -> CopyFromSchemaMapping {
798    let len = file.fields.len().min(table.fields.len());
799    let file_projection = (0..len).collect::<Vec<_>>();
800    let table_projection = (0..len).collect::<Vec<_>>();
801    let compatible_fields = file
802        .fields
803        .iter()
804        .enumerate()
805        .map(|(idx, file_field)| {
806            if idx < len {
807                table.fields[idx].clone()
808            } else {
809                file_field.clone()
810            }
811        })
812        .collect::<Vec<_>>();
813
814    CopyFromSchemaMapping {
815        file_projection,
816        table_projection,
817        compat_file_schema: Schema::new(compatible_fields),
818    }
819}
820
821#[cfg(test)]
822mod tests {
823    use std::sync::Arc;
824
825    use datatypes::arrow::datatypes::{DataType, Field, Schema};
826
827    use super::*;
828
829    fn test_schema_matches(from: (DataType, bool), to: (DataType, bool), matches: bool) {
830        let s1 = Arc::new(Schema::new(vec![Field::new("col", from.0.clone(), from.1)]));
831        let s2 = Arc::new(Schema::new(vec![Field::new("col", to.0.clone(), to.1)]));
832        let res = ensure_schema_compatible(&s1, &s2);
833        assert_eq!(
834            matches,
835            res.is_ok(),
836            "from data type: {}, to data type: {}, expected: {}, but got: {}",
837            from.0,
838            to.0,
839            matches,
840            res.is_ok()
841        )
842    }
843
844    #[test]
845    fn test_ensure_datatype_matches_ignore_timezone() {
846        test_schema_matches(
847            (
848                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
849                true,
850            ),
851            (
852                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
853                true,
854            ),
855            true,
856        );
857
858        test_schema_matches(
859            (
860                DataType::Timestamp(
861                    datatypes::arrow::datatypes::TimeUnit::Second,
862                    Some("UTC".into()),
863                ),
864                true,
865            ),
866            (
867                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
868                true,
869            ),
870            true,
871        );
872
873        test_schema_matches(
874            (
875                DataType::Timestamp(
876                    datatypes::arrow::datatypes::TimeUnit::Second,
877                    Some("UTC".into()),
878                ),
879                true,
880            ),
881            (
882                DataType::Timestamp(
883                    datatypes::arrow::datatypes::TimeUnit::Second,
884                    Some("PDT".into()),
885                ),
886                true,
887            ),
888            true,
889        );
890
891        test_schema_matches(
892            (
893                DataType::Timestamp(
894                    datatypes::arrow::datatypes::TimeUnit::Second,
895                    Some("UTC".into()),
896                ),
897                true,
898            ),
899            (
900                DataType::Timestamp(
901                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
902                    Some("UTC".into()),
903                ),
904                true,
905            ),
906            true,
907        );
908
909        test_schema_matches((DataType::Int8, true), (DataType::Int8, true), true);
910
911        test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
912    }
913
914    #[test]
915    fn test_data_type_equals_ignore_timezone_with_options() {
916        test_schema_matches(
917            (
918                DataType::Timestamp(
919                    datatypes::arrow::datatypes::TimeUnit::Microsecond,
920                    Some("UTC".into()),
921                ),
922                true,
923            ),
924            (
925                DataType::Timestamp(
926                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
927                    Some("PDT".into()),
928                ),
929                true,
930            ),
931            true,
932        );
933
934        test_schema_matches(
935            (DataType::Utf8, true),
936            (
937                DataType::Timestamp(
938                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
939                    Some("PDT".into()),
940                ),
941                true,
942            ),
943            true,
944        );
945
946        test_schema_matches(
947            (
948                DataType::Timestamp(
949                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
950                    Some("PDT".into()),
951                ),
952                true,
953            ),
954            (DataType::Utf8, true),
955            true,
956        );
957    }
958
959    #[test]
960    fn test_map_to_binary_json_compatibility() {
961        // Test Map -> Binary conversion for JSON types
962        let map_type = DataType::Map(
963            Arc::new(Field::new(
964                "key_value",
965                DataType::Struct(
966                    vec![
967                        Field::new("key", DataType::Utf8, false),
968                        Field::new("value", DataType::Utf8, false),
969                    ]
970                    .into(),
971                ),
972                false,
973            )),
974            false,
975        );
976
977        test_schema_matches((map_type, false), (DataType::Binary, true), true);
978
979        test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
980        test_schema_matches((DataType::Utf8, true), (DataType::Binary, true), true);
981    }
982
983    fn make_test_schema(v: &[Field]) -> Arc<Schema> {
984        Arc::new(Schema::new(v.to_vec()))
985    }
986
987    #[test]
988    fn test_compatible_file_schema() {
989        let file_schema0 = make_test_schema(&[
990            Field::new("c1", DataType::UInt8, true),
991            Field::new("c2", DataType::UInt8, true),
992        ]);
993
994        let table_schema = make_test_schema(&[
995            Field::new("c1", DataType::Int16, true),
996            Field::new("c2", DataType::Int16, true),
997            Field::new("c3", DataType::Int16, true),
998        ]);
999
1000        let compat_schema = make_test_schema(&[
1001            Field::new("c1", DataType::Int16, true),
1002            Field::new("c2", DataType::Int16, true),
1003        ]);
1004
1005        let (_, tp, _) =
1006            generated_schema_projection_and_compatible_file_schema(&file_schema0, &table_schema);
1007
1008        assert_eq!(table_schema.project(&tp).unwrap(), *compat_schema);
1009    }
1010
1011    #[test]
1012    fn test_schema_projection() {
1013        let file_schema0 = make_test_schema(&[
1014            Field::new("c1", DataType::UInt8, true),
1015            Field::new("c2", DataType::UInt8, true),
1016            Field::new("c3", DataType::UInt8, true),
1017        ]);
1018
1019        let file_schema1 = make_test_schema(&[
1020            Field::new("c3", DataType::UInt8, true),
1021            Field::new("c4", DataType::UInt8, true),
1022        ]);
1023
1024        let file_schema2 = make_test_schema(&[
1025            Field::new("c3", DataType::UInt8, true),
1026            Field::new("c4", DataType::UInt8, true),
1027            Field::new("c5", DataType::UInt8, true),
1028        ]);
1029
1030        let file_schema3 = make_test_schema(&[
1031            Field::new("c1", DataType::UInt8, true),
1032            Field::new("c2", DataType::UInt8, true),
1033        ]);
1034
1035        let table_schema = make_test_schema(&[
1036            Field::new("c3", DataType::UInt8, true),
1037            Field::new("c4", DataType::UInt8, true),
1038            Field::new("c5", DataType::UInt8, true),
1039        ]);
1040
1041        let tests = [
1042            (&file_schema0, &table_schema, true), // intersection
1043            (&file_schema1, &table_schema, true), // subset
1044            (&file_schema2, &table_schema, true), // full-eq
1045            (&file_schema3, &table_schema, true), // non-intersection
1046        ];
1047
1048        for test in tests {
1049            let (fp, tp, _) =
1050                generated_schema_projection_and_compatible_file_schema(test.0, test.1);
1051            assert_eq!(test.0.project(&fp).unwrap(), test.1.project(&tp).unwrap());
1052        }
1053    }
1054
1055    #[test]
1056    fn test_csv_reader_schema_for_skip_bad_records() {
1057        let file_schema = make_test_schema(&[
1058            Field::new("id", DataType::Utf8, true),
1059            Field::new("jsons", DataType::Utf8, true),
1060            Field::new("ts", DataType::Utf8, true),
1061        ]);
1062        let compat_schema = make_test_schema(&[
1063            Field::new("id", DataType::UInt32, true),
1064            Field::new("jsons", DataType::Binary, true),
1065            Field::new(
1066                "ts",
1067                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1068                true,
1069            ),
1070        ]);
1071
1072        let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1073
1074        assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1075        assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1076        assert_eq!(
1077            reader_schema.field(2).data_type(),
1078            compat_schema.field(2).data_type()
1079        );
1080    }
1081
1082    fn make_csv_metadata(schema: Arc<Schema>, has_header: bool) -> FileMetadata {
1083        FileMetadata::Csv {
1084            schema,
1085            format: CsvFormat {
1086                has_header,
1087                ..CsvFormat::default()
1088            },
1089            path: "test.csv".to_string(),
1090        }
1091    }
1092
1093    fn make_strict_csv_metadata(schema: Arc<Schema>) -> FileMetadata {
1094        FileMetadata::Csv {
1095            schema,
1096            format: CsvFormat {
1097                strict_headers: true,
1098                ..CsvFormat::default()
1099            },
1100            path: "test.csv".to_string(),
1101        }
1102    }
1103
1104    fn assert_field(schema: &Schema, idx: usize, name: &str, data_type: &DataType) {
1105        let field = schema.field(idx);
1106        assert_eq!(field.name(), name);
1107        assert_eq!(field.data_type(), data_type);
1108    }
1109
1110    #[test]
1111    fn test_strict_csv_headers_allows_reordered_columns() {
1112        let file_schema = make_test_schema(&[
1113            Field::new("ts", DataType::Utf8, true),
1114            Field::new("host_id", DataType::UInt8, true),
1115            Field::new("reading_value", DataType::Float64, true),
1116        ]);
1117        let table_schema = make_test_schema(&[
1118            Field::new("host_id", DataType::UInt32, true),
1119            Field::new("reading_value", DataType::Float64, true),
1120            Field::new("ts", DataType::Utf8, true),
1121        ]);
1122
1123        validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1124            .unwrap();
1125    }
1126
1127    #[test]
1128    fn test_strict_csv_headers_rejects_unknown_columns() {
1129        let file_schema = make_test_schema(&[
1130            Field::new("host_id", DataType::UInt8, true),
1131            Field::new("reading_value", DataType::Float64, true),
1132            Field::new("ts", DataType::Utf8, true),
1133            Field::new("extra", DataType::Utf8, true),
1134        ]);
1135        let table_schema = make_test_schema(&[
1136            Field::new("host_id", DataType::UInt32, true),
1137            Field::new("reading_value", DataType::Float64, true),
1138            Field::new("ts", DataType::Utf8, true),
1139        ]);
1140
1141        let err =
1142            validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1143                .unwrap_err();
1144
1145        assert!(matches!(
1146            err,
1147            error::Error::CsvHeaderMismatch {
1148                unknown_columns,
1149                missing_columns,
1150                duplicate_columns,
1151                ..
1152            } if unknown_columns == vec!["extra".to_string()]
1153                && missing_columns.is_empty()
1154                && duplicate_columns.is_empty()
1155        ));
1156    }
1157
1158    #[test]
1159    fn test_strict_csv_headers_rejects_missing_columns() {
1160        let file_schema = make_test_schema(&[
1161            Field::new("host_id", DataType::UInt8, true),
1162            Field::new("ts", DataType::Utf8, true),
1163        ]);
1164        let table_schema = make_test_schema(&[
1165            Field::new("host_id", DataType::UInt32, true),
1166            Field::new("reading_value", DataType::Float64, true),
1167            Field::new("ts", DataType::Utf8, true),
1168        ]);
1169
1170        let err =
1171            validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1172                .unwrap_err();
1173
1174        assert!(matches!(
1175            err,
1176            error::Error::CsvHeaderMismatch {
1177                unknown_columns,
1178                missing_columns,
1179                duplicate_columns,
1180                ..
1181            } if unknown_columns.is_empty()
1182                && missing_columns == vec!["reading_value".to_string()]
1183                && duplicate_columns.is_empty()
1184        ));
1185    }
1186
1187    #[test]
1188    fn test_strict_csv_headers_rejects_duplicate_columns() {
1189        let file_schema = make_test_schema(&[
1190            Field::new("host_id", DataType::UInt8, true),
1191            Field::new("reading_value", DataType::Float64, true),
1192            Field::new("ts", DataType::Utf8, true),
1193            Field::new("host_id", DataType::UInt16, true),
1194        ]);
1195        let table_schema = make_test_schema(&[
1196            Field::new("host_id", DataType::UInt32, true),
1197            Field::new("reading_value", DataType::Float64, true),
1198            Field::new("ts", DataType::Utf8, true),
1199        ]);
1200
1201        let err =
1202            validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1203                .unwrap_err();
1204
1205        assert!(matches!(
1206            err,
1207            error::Error::CsvHeaderMismatch {
1208                unknown_columns,
1209                missing_columns,
1210                duplicate_columns,
1211                ..
1212            } if unknown_columns.is_empty()
1213                && missing_columns.is_empty()
1214                && duplicate_columns == vec!["host_id".to_string()]
1215        ));
1216    }
1217
1218    #[test]
1219    fn test_headerless_csv_schema_projection_is_positional() {
1220        let file_schema = make_test_schema(&[
1221            Field::new("column_1", DataType::UInt8, true),
1222            Field::new("column_2", DataType::Float64, true),
1223            Field::new("column_3", DataType::Utf8, true),
1224        ]);
1225        let table_schema = make_test_schema(&[
1226            Field::new("host_id", DataType::UInt32, true),
1227            Field::new("reading_value", DataType::Float64, true),
1228            Field::new(
1229                "ts",
1230                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1231                true,
1232            ),
1233        ]);
1234
1235        let mapping =
1236            copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1237
1238        assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1239        assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1240        assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1241        assert_field(
1242            &mapping.compat_file_schema,
1243            1,
1244            "reading_value",
1245            &DataType::Float64,
1246        );
1247        assert_field(
1248            &mapping.compat_file_schema,
1249            2,
1250            "ts",
1251            table_schema.field(2).data_type(),
1252        );
1253        assert_eq!(
1254            mapping
1255                .compat_file_schema
1256                .project(&mapping.file_projection)
1257                .unwrap(),
1258            table_schema.project(&mapping.table_projection).unwrap()
1259        );
1260    }
1261
1262    #[test]
1263    fn test_headerless_csv_schema_projection_ignores_extra_file_columns() {
1264        let file_schema = make_test_schema(&[
1265            Field::new("column_1", DataType::UInt8, true),
1266            Field::new("column_2", DataType::Float64, true),
1267            Field::new("column_3", DataType::Utf8, true),
1268            Field::new("column_4", DataType::Utf8, true),
1269        ]);
1270        let table_schema = make_test_schema(&[
1271            Field::new("host_id", DataType::UInt32, true),
1272            Field::new("reading_value", DataType::Float64, true),
1273            Field::new("ts", DataType::Utf8, true),
1274        ]);
1275
1276        let mapping =
1277            copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1278
1279        assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1280        assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1281        assert_eq!(mapping.compat_file_schema.fields().len(), 4);
1282        assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1283        assert_field(
1284            &mapping.compat_file_schema,
1285            1,
1286            "reading_value",
1287            &DataType::Float64,
1288        );
1289        assert_field(&mapping.compat_file_schema, 2, "ts", &DataType::Utf8);
1290        assert_field(&mapping.compat_file_schema, 3, "column_4", &DataType::Utf8);
1291    }
1292
1293    #[test]
1294    fn test_headerless_csv_schema_projection_supports_prefix_import() {
1295        let file_schema = make_test_schema(&[
1296            Field::new("column_1", DataType::UInt8, true),
1297            Field::new("column_2", DataType::Float64, true),
1298        ]);
1299        let table_schema = make_test_schema(&[
1300            Field::new("host_id", DataType::UInt32, true),
1301            Field::new("reading_value", DataType::Float64, true),
1302            Field::new("ts", DataType::Utf8, true),
1303        ]);
1304
1305        let mapping =
1306            copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1307
1308        assert_eq!(mapping.file_projection, vec![0, 1]);
1309        assert_eq!(mapping.table_projection, vec![0, 1]);
1310        assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1311        assert_field(
1312            &mapping.compat_file_schema,
1313            1,
1314            "reading_value",
1315            &DataType::Float64,
1316        );
1317        assert_eq!(
1318            mapping
1319                .compat_file_schema
1320                .project(&mapping.file_projection)
1321                .unwrap(),
1322            table_schema.project(&mapping.table_projection).unwrap()
1323        );
1324    }
1325
1326    #[test]
1327    fn test_csv_reader_schema_for_skip_bad_records_uses_positional_mapping() {
1328        let file_schema = make_test_schema(&[
1329            Field::new("column_1", DataType::Utf8, true),
1330            Field::new("column_2", DataType::Utf8, true),
1331            Field::new("column_3", DataType::Utf8, true),
1332        ]);
1333        let table_schema = make_test_schema(&[
1334            Field::new("host_id", DataType::UInt32, true),
1335            Field::new("jsons", DataType::Binary, true),
1336            Field::new(
1337                "ts",
1338                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1339                true,
1340            ),
1341        ]);
1342        let mapping = copy_from_schema_mapping(
1343            &make_csv_metadata(file_schema.clone(), false),
1344            &table_schema,
1345        );
1346        let compat_schema = Arc::new(mapping.compat_file_schema);
1347
1348        let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1349
1350        assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1351        assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1352        assert_eq!(
1353            reader_schema.field(2).data_type(),
1354            table_schema.field(2).data_type()
1355        );
1356    }
1357}