Skip to main content

operator/statement/
copy_table_from.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeSet, HashMap, HashSet};
16use std::future::Future;
17use std::pin::Pin;
18use std::sync::Arc;
19use std::task::{Context, Poll};
20
21use client::{Output, OutputData, OutputMeta};
22use common_base::readable_size::ReadableSize;
23use common_datasource::file_format::csv::{
24    CsvFormat, is_skippable_arrow_error, tolerant_csv_stream,
25};
26use common_datasource::file_format::json::JsonFormat;
27use common_datasource::file_format::orc::{ReaderAdapter, infer_orc_schema, new_orc_stream_reader};
28use common_datasource::file_format::{FileFormat, Format, file_to_stream};
29use common_datasource::lister::{Lister, Source};
30use common_datasource::object_store::{LocalFileAccess, build_backend_with_path};
31use common_query::{OutputCost, OutputRows};
32use common_recordbatch::DfSendableRecordBatchStream;
33use common_recordbatch::adapter::RecordBatchStreamTypeAdapter;
34use common_telemetry::{debug, tracing};
35use datafusion::datasource::physical_plan::{CsvSource, FileSource, JsonSource};
36use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
37use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata;
38use datafusion_common::DataFusionError;
39use datafusion_common::arrow::error::ArrowError;
40use datafusion_common::config::CsvOptions;
41use datafusion_expr::Expr;
42use datatypes::arrow::compute::can_cast_types;
43use datatypes::arrow::datatypes::{DataType as ArrowDataType, Schema, SchemaRef};
44use datatypes::arrow::record_batch::RecordBatch;
45use datatypes::vectors::Helper;
46use futures_util::StreamExt;
47use object_store::{EntryMode, ObjectStore};
48use regex::Regex;
49use session::context::QueryContextRef;
50use snafu::{ResultExt, ensure};
51use table::requests::{CopyTableRequest, InsertRequest};
52use table::table_reference::TableReference;
53use tokio_util::compat::FuturesAsyncReadCompatExt;
54
55use crate::error::{self, IntoVectorsSnafu, Result};
56use crate::statement::StatementExecutor;
57
58const DEFAULT_BATCH_SIZE: usize = 8192;
59const DEFAULT_READ_BUFFER: usize = 256 * 1024;
60
61enum FileMetadata {
62    Parquet {
63        schema: SchemaRef,
64        metadata: ArrowReaderMetadata,
65        path: String,
66    },
67    Orc {
68        schema: SchemaRef,
69        path: String,
70    },
71    Json {
72        schema: SchemaRef,
73        format: JsonFormat,
74        path: String,
75    },
76    Csv {
77        schema: SchemaRef,
78        format: CsvFormat,
79        path: String,
80    },
81}
82
83impl FileMetadata {
84    /// Returns the [SchemaRef]
85    pub fn schema(&self) -> &SchemaRef {
86        match self {
87            FileMetadata::Parquet { schema, .. } => schema,
88            FileMetadata::Orc { schema, .. } => schema,
89            FileMetadata::Json { schema, .. } => schema,
90            FileMetadata::Csv { schema, .. } => schema,
91        }
92    }
93}
94
95async fn list_copy_from_paths(
96    req: &CopyTableRequest,
97    local_file_access: &LocalFileAccess,
98) -> Result<(ObjectStore, Vec<String>)> {
99    let backend = build_backend_with_path(&req.location, &req.connection, local_file_access)
100        .await
101        .context(error::BuildBackendSnafu)?;
102    let regex = req
103        .pattern
104        .as_ref()
105        .map(|x| Regex::new(x))
106        .transpose()
107        .context(error::BuildRegexSnafu)?;
108
109    // Listing a known file's parent for every table makes COPY DATABASE do
110    // quadratic directory work.
111    if let Some(filename) = &backend.object_path {
112        let is_file = backend
113            .is_file(filename)
114            .await
115            .with_context(|_| common_datasource::error::ListObjectsSnafu {
116                path: req.location.clone(),
117            })
118            .context(error::ListObjectsSnafu)?;
119        let paths = if is_file {
120            vec![filename.clone()]
121        } else {
122            vec![]
123        };
124        return Ok((backend.object_store, paths));
125    }
126
127    let lister = Lister::new(
128        backend.object_store.clone(),
129        Source::Dir,
130        req.location.clone(),
131        regex,
132    );
133
134    let entries = lister.list().await.context(error::ListObjectsSnafu)?;
135    debug!(
136        "Copy from location: {:?}, entries: {entries:?}",
137        req.location
138    );
139    let paths = entries
140        .into_iter()
141        .filter(|entry| entry.metadata().mode() == EntryMode::FILE)
142        .map(|entry| entry.path().to_string())
143        .collect();
144    Ok((backend.object_store, paths))
145}
146
147impl StatementExecutor {
148    async fn collect_metadata(
149        &self,
150        object_store: &ObjectStore,
151        format: Format,
152        path: String,
153    ) -> Result<FileMetadata> {
154        match format {
155            Format::Csv(format) => Ok(FileMetadata::Csv {
156                schema: Arc::new(
157                    format
158                        .infer_schema(object_store, &path)
159                        .await
160                        .context(error::InferSchemaSnafu { path: &path })?,
161                ),
162                format,
163                path,
164            }),
165            Format::Json(format) => Ok(FileMetadata::Json {
166                schema: Arc::new(
167                    format
168                        .infer_schema(object_store, &path)
169                        .await
170                        .context(error::InferSchemaSnafu { path: &path })?,
171                ),
172                format,
173                path,
174            }),
175            Format::Parquet(_) => {
176                let meta = object_store
177                    .stat(&path)
178                    .await
179                    .context(error::ReadObjectSnafu { path: &path })?;
180                let mut reader = object_store
181                    .reader(&path)
182                    .await
183                    .context(error::ReadObjectSnafu { path: &path })?
184                    .into_futures_async_read(0..meta.content_length())
185                    .await
186                    .context(error::ReadObjectSnafu { path: &path })?
187                    .compat();
188                let metadata = ArrowReaderMetadata::load_async(&mut reader, Default::default())
189                    .await
190                    .context(error::ReadParquetMetadataSnafu)?;
191
192                Ok(FileMetadata::Parquet {
193                    schema: metadata.schema().clone(),
194                    metadata,
195                    path,
196                })
197            }
198            Format::Orc(_) => {
199                let meta = object_store
200                    .stat(&path)
201                    .await
202                    .context(error::ReadObjectSnafu { path: &path })?;
203
204                let reader = object_store
205                    .reader(&path)
206                    .await
207                    .context(error::ReadObjectSnafu { path: &path })?;
208
209                let schema = infer_orc_schema(ReaderAdapter::new(reader, meta.content_length()))
210                    .await
211                    .context(error::ReadOrcSnafu)?;
212
213                Ok(FileMetadata::Orc {
214                    schema: Arc::new(schema),
215                    path,
216                })
217            }
218        }
219    }
220
221    async fn build_read_stream(
222        &self,
223        compat_schema: SchemaRef,
224        object_store: &ObjectStore,
225        file_metadata: &FileMetadata,
226        projection: Vec<usize>,
227        filters: Vec<Expr>,
228    ) -> Result<DfSendableRecordBatchStream> {
229        match file_metadata {
230            FileMetadata::Csv {
231                format,
232                path,
233                schema,
234            } => {
235                let output_schema = Arc::new(
236                    compat_schema
237                        .project(&projection)
238                        .context(error::ProjectSchemaSnafu)?,
239                );
240
241                let options = CsvOptions::default()
242                    .with_has_header(format.has_header)
243                    .with_delimiter(format.delimiter);
244                let csv_source = CsvSource::new(schema.clone())
245                    .with_csv_options(options)
246                    .with_batch_size(DEFAULT_BATCH_SIZE);
247                let stream = if format.skip_bad_records {
248                    let reader_schema =
249                        csv_reader_schema_for_skip_bad_records(schema, &compat_schema);
250                    tolerant_csv_stream(
251                        object_store,
252                        path,
253                        Arc::new(reader_schema),
254                        projection.clone(),
255                        format,
256                    )
257                    .await
258                    .context(error::BuildFileStreamSnafu)?
259                } else {
260                    file_to_stream(
261                        object_store,
262                        path,
263                        csv_source,
264                        Some(projection),
265                        format.compression_type,
266                    )
267                    .await
268                    .context(error::BuildFileStreamSnafu)?
269                };
270
271                let stream = Box::pin(
272                    // The projection is already applied in the CSV reader when we created the stream,
273                    // so we pass None here to avoid double projection which would cause schema mismatch errors.
274                    RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
275                        .with_filter(filters)
276                        .context(error::PhysicalExprSnafu)?,
277                );
278                if format.skip_bad_records {
279                    Ok(Box::pin(SkipBadRecordsStream::new(stream, path)))
280                } else {
281                    Ok(stream)
282                }
283            }
284            FileMetadata::Json {
285                path,
286                format,
287                schema,
288            } => {
289                let output_schema = Arc::new(
290                    compat_schema
291                        .project(&projection)
292                        .context(error::ProjectSchemaSnafu)?,
293                );
294
295                let json_source =
296                    JsonSource::new(schema.clone()).with_batch_size(DEFAULT_BATCH_SIZE);
297                let stream = file_to_stream(
298                    object_store,
299                    path,
300                    json_source,
301                    Some(projection),
302                    format.compression_type,
303                )
304                .await
305                .context(error::BuildFileStreamSnafu)?;
306
307                Ok(Box::pin(
308                    // The projection is already applied in the JSON reader when we created the stream,
309                    // so we pass None here to avoid double projection which would cause schema mismatch errors.
310                    RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
311                        .with_filter(filters)
312                        .context(error::PhysicalExprSnafu)?,
313                ))
314            }
315            FileMetadata::Parquet { metadata, path, .. } => {
316                let meta = object_store
317                    .stat(path)
318                    .await
319                    .context(error::ReadObjectSnafu { path })?;
320                let reader = object_store
321                    .reader_with(path)
322                    .chunk(DEFAULT_READ_BUFFER)
323                    .await
324                    .context(error::ReadObjectSnafu { path })?
325                    .into_futures_async_read(0..meta.content_length())
326                    .await
327                    .context(error::ReadObjectSnafu { path })?
328                    .compat();
329                let builder =
330                    ParquetRecordBatchStreamBuilder::new_with_metadata(reader, metadata.clone());
331                let stream = builder
332                    .build()
333                    .context(error::BuildParquetRecordBatchStreamSnafu)?;
334
335                let output_schema = Arc::new(
336                    compat_schema
337                        .project(&projection)
338                        .context(error::ProjectSchemaSnafu)?,
339                );
340                Ok(Box::pin(
341                    RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
342                        .with_filter(filters)
343                        .context(error::PhysicalExprSnafu)?,
344                ))
345            }
346            FileMetadata::Orc { path, .. } => {
347                let meta = object_store
348                    .stat(path)
349                    .await
350                    .context(error::ReadObjectSnafu { path })?;
351
352                let reader = object_store
353                    .reader_with(path)
354                    .chunk(DEFAULT_READ_BUFFER)
355                    .await
356                    .context(error::ReadObjectSnafu { path })?;
357                let stream =
358                    new_orc_stream_reader(ReaderAdapter::new(reader, meta.content_length()))
359                        .await
360                        .context(error::ReadOrcSnafu)?;
361
362                let output_schema = Arc::new(
363                    compat_schema
364                        .project(&projection)
365                        .context(error::ProjectSchemaSnafu)?,
366                );
367
368                Ok(Box::pin(
369                    RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
370                        .with_filter(filters)
371                        .context(error::PhysicalExprSnafu)?,
372                ))
373            }
374        }
375    }
376
377    #[tracing::instrument(skip_all)]
378    pub async fn copy_table_from(
379        &self,
380        req: CopyTableRequest,
381        query_ctx: QueryContextRef,
382    ) -> Result<Output> {
383        let table_ref = TableReference {
384            catalog: &req.catalog_name,
385            schema: &req.schema_name,
386            table: &req.table_name,
387        };
388        let table = self.get_table(&table_ref).await?;
389        let format = Format::try_from(&req.with).context(error::ParseFileFormatSnafu)?;
390        let (object_store, paths) = list_copy_from_paths(&req, &self.local_file_access).await?;
391        let mut files = Vec::with_capacity(paths.len());
392        let table_schema = table.schema().arrow_schema().clone();
393        let filters = table
394            .schema()
395            .timestamp_column()
396            .and_then(|c| {
397                common_query::logical_plan::build_same_type_ts_filter(c, req.timestamp_range)
398            })
399            .into_iter()
400            .collect::<Vec<_>>();
401
402        for path in paths {
403            let file_metadata = self
404                .collect_metadata(&object_store, format.clone(), path)
405                .await?;
406
407            validate_csv_headers_if_required(&file_metadata, &table_schema)?;
408            let schema_mapping = copy_from_schema_mapping(&file_metadata, &table_schema);
409            let projected_file_schema = Arc::new(
410                file_metadata
411                    .schema()
412                    .project(&schema_mapping.file_projection)
413                    .context(error::ProjectSchemaSnafu)?,
414            );
415            let projected_table_schema = Arc::new(
416                table_schema
417                    .project(&schema_mapping.table_projection)
418                    .context(error::ProjectSchemaSnafu)?,
419            );
420            ensure_schema_compatible(&projected_file_schema, &projected_table_schema)?;
421
422            files.push((
423                Arc::new(schema_mapping.compat_file_schema),
424                schema_mapping.file_projection,
425                projected_table_schema,
426                file_metadata,
427            ))
428        }
429
430        let mut rows_inserted = 0;
431        let mut insert_cost = 0;
432        let max_insert_rows = req
433            .limit
434            .map(|n| {
435                usize::try_from(n).map_err(|_| {
436                    error::InvalidCopyParameterSnafu {
437                        key: "limit".to_string(),
438                        value: n.to_string(),
439                    }
440                    .build()
441                })
442            })
443            .transpose()?;
444        if max_insert_rows == Some(0) {
445            return Ok(gen_insert_output(rows_inserted, insert_cost));
446        }
447
448        let mut accepted_rows = 0;
449        for (compat_schema, file_schema_projection, projected_table_schema, file_metadata) in files
450        {
451            let mut stream = self
452                .build_read_stream(
453                    compat_schema,
454                    &object_store,
455                    &file_metadata,
456                    file_schema_projection,
457                    filters.clone(),
458                )
459                .await?;
460
461            let fields = projected_table_schema
462                .fields()
463                .iter()
464                .map(|f| f.name().clone())
465                .collect::<Vec<_>>();
466
467            // TODO(hl): make this configurable through options.
468            let pending_mem_threshold = ReadableSize::mb(32).as_bytes();
469            let mut pending_mem_size = 0;
470            let mut pending = vec![];
471
472            while let Some(r) = stream.next().await {
473                let record_batch = r.context(error::ReadDfRecordBatchSnafu)?;
474                let record_batch = if let Some(max_insert_rows) = max_insert_rows {
475                    let remaining_rows = max_insert_rows - accepted_rows;
476                    if record_batch.num_rows() > remaining_rows {
477                        record_batch.slice(0, remaining_rows)
478                    } else {
479                        record_batch
480                    }
481                } else {
482                    record_batch
483                };
484                let record_batch_rows = record_batch.num_rows();
485                let vectors =
486                    Helper::try_into_vectors(record_batch.columns()).context(IntoVectorsSnafu)?;
487
488                pending_mem_size += vectors.iter().map(|v| v.memory_size()).sum::<usize>();
489
490                let columns_values = fields
491                    .iter()
492                    .cloned()
493                    .zip(vectors)
494                    .collect::<HashMap<_, _>>();
495
496                pending.push(self.inserter.handle_table_insert(
497                    InsertRequest {
498                        catalog_name: req.catalog_name.clone(),
499                        schema_name: req.schema_name.clone(),
500                        table_name: req.table_name.clone(),
501                        columns_values,
502                        skip_wal: query_ctx.skip_wal(),
503                    },
504                    query_ctx.clone(),
505                ));
506                accepted_rows += record_batch_rows;
507
508                if pending_mem_size as u64 >= pending_mem_threshold {
509                    let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
510                    rows_inserted += rows;
511                    insert_cost += cost;
512                }
513
514                if let Some(max_insert_rows) = max_insert_rows
515                    && accepted_rows == max_insert_rows
516                {
517                    if !pending.is_empty() {
518                        let (rows, cost) =
519                            batch_insert(&mut pending, &mut pending_mem_size).await?;
520                        rows_inserted += rows;
521                        insert_cost += cost;
522                    }
523                    return Ok(gen_insert_output(rows_inserted, insert_cost));
524                }
525            }
526
527            if !pending.is_empty() {
528                let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
529                rows_inserted += rows;
530                insert_cost += cost;
531            }
532        }
533
534        Ok(gen_insert_output(rows_inserted, insert_cost))
535    }
536}
537
538fn gen_insert_output(rows_inserted: usize, insert_cost: usize) -> Output {
539    Output::new(
540        OutputData::AffectedRows(rows_inserted),
541        OutputMeta::new_with_cost(insert_cost),
542    )
543}
544
545struct SkipBadRecordsStream {
546    inner: DfSendableRecordBatchStream,
547    path: String,
548}
549
550impl SkipBadRecordsStream {
551    fn new(inner: DfSendableRecordBatchStream, path: impl Into<String>) -> Self {
552        Self {
553            inner,
554            path: path.into(),
555        }
556    }
557}
558
559impl datafusion::physical_plan::RecordBatchStream for SkipBadRecordsStream {
560    fn schema(&self) -> SchemaRef {
561        self.inner.schema()
562    }
563}
564
565impl futures::Stream for SkipBadRecordsStream {
566    type Item = datafusion_common::Result<RecordBatch>;
567
568    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
569        let this = self.get_mut();
570        loop {
571            match this.inner.as_mut().poll_next(cx) {
572                Poll::Ready(Some(Err(error))) if is_skippable_record_error(&error) => {
573                    common_telemetry::warn!(
574                        "Skipping bad record while copying from {}: {}",
575                        this.path,
576                        error
577                    );
578                    continue;
579                }
580                other => return other,
581            }
582        }
583    }
584}
585
586fn is_skippable_record_error(error: &DataFusionError) -> bool {
587    match error {
588        DataFusionError::ArrowError(error, _) => is_skippable_arrow_error(error),
589        DataFusionError::External(error) => error
590            .downcast_ref::<ArrowError>()
591            .is_some_and(is_skippable_arrow_error),
592        DataFusionError::Context(_, error) => is_skippable_record_error(error),
593        _ => false,
594    }
595}
596
597/// Executes all pending inserts all at once, drain pending requests and reset pending bytes.
598async fn batch_insert(
599    pending: &mut Vec<impl Future<Output = Result<Output>>>,
600    pending_bytes: &mut usize,
601) -> Result<(OutputRows, OutputCost)> {
602    let batch = pending.drain(..);
603    let result = futures::future::try_join_all(batch)
604        .await?
605        .iter()
606        .map(|o| o.extract_rows_and_cost())
607        .reduce(|(a, b), (c, d)| (a + c, b + d))
608        .unwrap_or((0, 0));
609    *pending_bytes = 0;
610    Ok(result)
611}
612
613/// Custom type compatibility check for GreptimeDB that handles Map -> Binary (JSON) conversion
614fn can_cast_types_for_greptime(from: &ArrowDataType, to: &ArrowDataType) -> bool {
615    // Handle Map -> Binary conversion for JSON types
616    if let ArrowDataType::Map(_, _) = from
617        && let ArrowDataType::Binary = to
618    {
619        return true;
620    }
621
622    // For all other cases, use Arrow's built-in can_cast_types
623    can_cast_types(from, to)
624}
625
626fn csv_reader_schema_for_skip_bad_records(file: &SchemaRef, compat: &SchemaRef) -> Schema {
627    let fields = file
628        .fields()
629        .iter()
630        .enumerate()
631        .map(|(idx, file_field)| match compat.fields().get(idx) {
632            Some(compat_field) if can_csv_reader_parse_type(compat_field.data_type()) => {
633                compat_field.clone()
634            }
635            _ => file_field.clone(),
636        })
637        .collect::<Vec<_>>();
638
639    Schema::new_with_metadata(fields, file.metadata().clone())
640}
641
642fn can_csv_reader_parse_type(data_type: &ArrowDataType) -> bool {
643    match data_type {
644        ArrowDataType::Boolean
645        | ArrowDataType::Decimal32(_, _)
646        | ArrowDataType::Decimal64(_, _)
647        | ArrowDataType::Decimal128(_, _)
648        | ArrowDataType::Decimal256(_, _)
649        | ArrowDataType::Int8
650        | ArrowDataType::Int16
651        | ArrowDataType::Int32
652        | ArrowDataType::Int64
653        | ArrowDataType::UInt8
654        | ArrowDataType::UInt16
655        | ArrowDataType::UInt32
656        | ArrowDataType::UInt64
657        | ArrowDataType::Float32
658        | ArrowDataType::Float64
659        | ArrowDataType::Date32
660        | ArrowDataType::Date64
661        | ArrowDataType::Time32(_)
662        | ArrowDataType::Time64(_)
663        | ArrowDataType::Timestamp(_, _)
664        | ArrowDataType::Null
665        | ArrowDataType::Utf8
666        | ArrowDataType::Utf8View => true,
667        ArrowDataType::Dictionary(_, value_type) => value_type.as_ref() == &ArrowDataType::Utf8,
668        _ => false,
669    }
670}
671
672fn ensure_schema_compatible(from: &SchemaRef, to: &SchemaRef) -> Result<()> {
673    let not_match = from
674        .fields
675        .iter()
676        .zip(to.fields.iter())
677        .map(|(l, r)| (l.data_type(), r.data_type()))
678        .enumerate()
679        .find(|(_, (l, r))| !can_cast_types_for_greptime(l, r));
680
681    if let Some((index, _)) = not_match {
682        error::InvalidSchemaSnafu {
683            index,
684            table_schema: to.to_string(),
685            file_schema: from.to_string(),
686        }
687        .fail()
688    } else {
689        Ok(())
690    }
691}
692
693fn validate_csv_headers_if_required(file_metadata: &FileMetadata, table: &SchemaRef) -> Result<()> {
694    let FileMetadata::Csv {
695        schema,
696        format,
697        path,
698    } = file_metadata
699    else {
700        return Ok(());
701    };
702
703    if !format.strict_headers {
704        return Ok(());
705    }
706
707    let mut seen_file_columns = HashSet::with_capacity(schema.fields().len());
708    let duplicate_columns = schema
709        .fields()
710        .iter()
711        .filter_map(|field| {
712            if seen_file_columns.insert(field.name().clone()) {
713                None
714            } else {
715                Some(field.name().clone())
716            }
717        })
718        .collect::<BTreeSet<_>>()
719        .into_iter()
720        .collect::<Vec<_>>();
721    let file_columns = seen_file_columns.into_iter().collect::<BTreeSet<_>>();
722    let table_columns = table
723        .fields()
724        .iter()
725        .map(|field| field.name().clone())
726        .collect::<BTreeSet<_>>();
727    let unknown_columns = file_columns
728        .difference(&table_columns)
729        .cloned()
730        .collect::<Vec<_>>();
731    let missing_columns = table_columns
732        .difference(&file_columns)
733        .cloned()
734        .collect::<Vec<_>>();
735
736    ensure!(
737        unknown_columns.is_empty() && missing_columns.is_empty() && duplicate_columns.is_empty(),
738        error::CsvHeaderMismatchSnafu {
739            path,
740            unknown_columns,
741            missing_columns,
742            duplicate_columns,
743        }
744    );
745
746    Ok(())
747}
748
749/// Generates a maybe compatible schema of the file schema.
750///
751/// If there is a field is found in table schema,
752/// copy the field data type to maybe compatible schema(`compatible_fields`).
753fn generated_schema_projection_and_compatible_file_schema(
754    file: &SchemaRef,
755    table: &SchemaRef,
756) -> (Vec<usize>, Vec<usize>, Schema) {
757    let mut file_projection = Vec::with_capacity(file.fields.len());
758    let mut table_projection = Vec::with_capacity(file.fields.len());
759    let mut compatible_fields = file.fields.iter().cloned().collect::<Vec<_>>();
760    for (file_idx, file_field) in file.fields.iter().enumerate() {
761        if let Some((table_idx, table_field)) = table.fields.find(file_field.name()) {
762            file_projection.push(file_idx);
763            table_projection.push(table_idx);
764
765            // Safety: the compatible_fields has same length as file schema
766            compatible_fields[file_idx] = table_field.clone();
767        }
768    }
769
770    (
771        file_projection,
772        table_projection,
773        Schema::new(compatible_fields),
774    )
775}
776
777struct CopyFromSchemaMapping {
778    file_projection: Vec<usize>,
779    table_projection: Vec<usize>,
780    compat_file_schema: Schema,
781}
782
783fn copy_from_schema_mapping(
784    file_metadata: &FileMetadata,
785    table: &SchemaRef,
786) -> CopyFromSchemaMapping {
787    match file_metadata {
788        FileMetadata::Csv { schema, format, .. } if !format.has_header => {
789            generated_positional_schema_projection_and_compatible_file_schema(schema, table)
790        }
791        _ => {
792            let (file_projection, table_projection, compat_file_schema) =
793                generated_schema_projection_and_compatible_file_schema(
794                    file_metadata.schema(),
795                    table,
796                );
797            CopyFromSchemaMapping {
798                file_projection,
799                table_projection,
800                compat_file_schema,
801            }
802        }
803    }
804}
805
806fn generated_positional_schema_projection_and_compatible_file_schema(
807    file: &SchemaRef,
808    table: &SchemaRef,
809) -> CopyFromSchemaMapping {
810    let len = file.fields.len().min(table.fields.len());
811    let file_projection = (0..len).collect::<Vec<_>>();
812    let table_projection = (0..len).collect::<Vec<_>>();
813    let compatible_fields = file
814        .fields
815        .iter()
816        .enumerate()
817        .map(|(idx, file_field)| {
818            if idx < len {
819                table.fields[idx].clone()
820            } else {
821                file_field.clone()
822            }
823        })
824        .collect::<Vec<_>>();
825
826    CopyFromSchemaMapping {
827        file_projection,
828        table_projection,
829        compat_file_schema: Schema::new(compatible_fields),
830    }
831}
832
833#[cfg(test)]
834mod tests {
835    use std::sync::Arc;
836
837    use datatypes::arrow::datatypes::{DataType, Field, Schema};
838
839    use super::*;
840
841    fn copy_from_request(location: &str, pattern: Option<&str>) -> CopyTableRequest {
842        CopyTableRequest {
843            catalog_name: "greptime".into(),
844            schema_name: "public".into(),
845            table_name: "test".into(),
846            location: location.into(),
847            with: HashMap::new(),
848            connection: HashMap::new(),
849            pattern: pattern.map(String::from),
850            direction: table::requests::CopyDirection::Import,
851            timestamp_range: None,
852            limit: None,
853        }
854    }
855
856    #[tokio::test]
857    async fn test_copy_from_local_paths() {
858        use common_test_util::temp_dir::create_temp_dir;
859
860        let dir = create_temp_dir("copy_from_paths");
861        let nested = dir.path().join("nested");
862        std::fs::create_dir_all(nested.join("data.dir")).unwrap();
863        std::fs::write(nested.join("data file.parquet"), b"data").unwrap();
864        std::fs::write(nested.join("other.csv"), b"other").unwrap();
865        let access = LocalFileAccess::sandboxed(dir.path()).unwrap();
866        for location in [
867            "nested/data file.parquet".to_string(),
868            url::Url::from_file_path(nested.join("data file.parquet"))
869                .unwrap()
870                .to_string(),
871        ] {
872            // PATTERN applies to directories, not an explicitly selected file.
873            let req = copy_from_request(&location, Some("does-not-match"));
874            let (store, paths) = list_copy_from_paths(&req, &access).await.unwrap();
875            assert_eq!(paths, ["data file.parquet"]);
876            assert_eq!(store.read(&paths[0]).await.unwrap().to_vec(), b"data");
877        }
878
879        let req = copy_from_request("nested/", Some("^data"));
880        let (_, paths) = list_copy_from_paths(&req, &access).await.unwrap();
881        assert_eq!(paths, ["data file.parquet"]);
882        let req = copy_from_request("nested/", None);
883        let (_, mut paths) = list_copy_from_paths(&req, &access).await.unwrap();
884        paths.sort();
885        assert_eq!(paths, ["data file.parquet", "other.csv"]);
886        let req = copy_from_request("nested/data.dir", None);
887        assert!(
888            list_copy_from_paths(&req, &access)
889                .await
890                .unwrap()
891                .1
892                .is_empty()
893        );
894        let req = copy_from_request("nested/", Some("does-not-match"));
895        assert!(
896            list_copy_from_paths(&req, &access)
897                .await
898                .unwrap()
899                .1
900                .is_empty()
901        );
902
903        let req = copy_from_request("nested/missing.parquet", None);
904        assert!(matches!(
905            list_copy_from_paths(&req, &access).await,
906            Err(error::Error::ListObjects {
907                source: common_datasource::error::Error::ListObjects { error, .. }, ..
908            }) if error.kind() == object_store::ErrorKind::NotFound
909        ));
910        let req = copy_from_request("nested/data file.parquet", Some("["));
911        assert!(matches!(
912            list_copy_from_paths(&req, &access).await,
913            Err(error::Error::BuildRegex { .. })
914        ));
915        let req = copy_from_request("nested/data file.parquet", None);
916        assert!(matches!(
917            list_copy_from_paths(&req, &LocalFileAccess::Disabled).await,
918            Err(error::Error::BuildBackend {
919                source: common_datasource::error::Error::LocalFileAccessDisabled { .. },
920                ..
921            })
922        ));
923        let req = copy_from_request("../outside.parquet", None);
924        assert!(matches!(
925            list_copy_from_paths(&req, &access).await,
926            Err(error::Error::BuildBackend {
927                source: common_datasource::error::Error::LocalFileAccessDenied { .. },
928                ..
929            })
930        ));
931    }
932
933    #[cfg(unix)]
934    #[tokio::test]
935    async fn test_copy_from_local_symlinks() {
936        use common_test_util::temp_dir::create_temp_dir;
937
938        let dir = create_temp_dir("copy_from_symlinks");
939        std::fs::write(dir.path().join("data.parquet"), b"data").unwrap();
940        std::os::unix::fs::symlink("data.parquet", dir.path().join("link.parquet")).unwrap();
941        std::os::unix::fs::symlink("missing.parquet", dir.path().join("dangling.parquet")).unwrap();
942        let access = LocalFileAccess::sandboxed(dir.path()).unwrap();
943
944        let req = copy_from_request("link.parquet", None);
945        assert!(
946            list_copy_from_paths(&req, &access)
947                .await
948                .unwrap()
949                .1
950                .is_empty()
951        );
952        let req = copy_from_request("./", None);
953        let (_, paths) = list_copy_from_paths(&req, &access).await.unwrap();
954        assert_eq!(paths, ["data.parquet"]);
955        let req = copy_from_request("dangling.parquet", None);
956        assert!(matches!(
957            list_copy_from_paths(&req, &access).await,
958            Err(error::Error::ListObjects {
959                source: common_datasource::error::Error::ListObjects { error, .. }, ..
960            }) if error.kind() == object_store::ErrorKind::NotFound
961        ));
962    }
963
964    #[tokio::test]
965    async fn test_copy_from_s3_paths_without_relisting() {
966        use std::sync::Mutex;
967
968        use axum::Router;
969        use axum::http::{Method, StatusCode, Uri};
970        use axum::response::IntoResponse;
971
972        let requests = Arc::new(Mutex::new(Vec::new()));
973        let recorded = requests.clone();
974        let app = Router::new().fallback(move |method: Method, uri: Uri| {
975            let recorded = recorded.clone();
976            async move {
977                recorded.lock().unwrap().push((method.clone(), uri.clone()));
978                if method == Method::HEAD {
979                    let status = if uri.path().ends_with("missing.parquet") {
980                        StatusCode::NOT_FOUND
981                    } else if uri.path().ends_with("denied.parquet") {
982                        StatusCode::FORBIDDEN
983                    } else {
984                        StatusCode::OK
985                    };
986                    return (status, [("content-length", "0")]).into_response();
987                }
988                assert_eq!(method, Method::GET);
989                let query =
990                    axum::extract::Query::<HashMap<String, String>>::try_from_uri(&uri).unwrap();
991                assert_eq!(query.get("prefix").unwrap(), "backup/schema/1/");
992                assert_eq!(query.get("list-type").unwrap(), "2");
993                let files = (0..16).map(|i| format!(
994                    "<Contents><Key>backup/schema/1/data.{i}.parquet</Key><LastModified>2026-09-01T00:00:00Z</LastModified><Size>0</Size></Contents>"
995                )).collect::<String>();
996                format!(
997                    "<ListBucketResult><IsTruncated>false</IsTruncated>{files}\
998                    <Contents><Key>backup/schema/1/other.csv</Key><LastModified>2026-09-01T00:00:00Z</LastModified><Size>0</Size></Contents>\
999                    <CommonPrefixes><Prefix>backup/schema/1/data.dir/</Prefix></CommonPrefixes>\
1000                    </ListBucketResult>"
1001                )
1002                .into_response()
1003            }
1004        });
1005        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1006        let endpoint = format!("http://{}", listener.local_addr().unwrap());
1007        let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
1008        let connection = HashMap::from([
1009            ("endpoint".into(), endpoint),
1010            ("region".into(), "us-east-1".into()),
1011            ("access_key_id".into(), "test".into()),
1012            ("secret_access_key".into(), "test".into()),
1013            ("disable_ec2_metadata".into(), "true".into()),
1014        ]);
1015        for i in 0..16 {
1016            let filename = format!("data.{i}.parquet");
1017            let mut req = copy_from_request(
1018                &format!("s3://bucket/backup/schema/1/{filename}"),
1019                Some("does-not-match"),
1020            );
1021            req.connection = connection.clone();
1022            let (_, paths) = list_copy_from_paths(&req, &LocalFileAccess::Disabled)
1023                .await
1024                .unwrap();
1025            assert_eq!(paths, [filename]);
1026        }
1027        {
1028            let requests = requests.lock().unwrap();
1029            assert_eq!(requests.len(), 16);
1030            assert!(requests.iter().all(|(method, uri)| method == Method::HEAD
1031                && uri.path().starts_with("/bucket/backup/schema/1/")));
1032        }
1033        let mut req = copy_from_request("s3://bucket/backup/schema/1/", Some(r"^data\."));
1034        req.connection = connection.clone();
1035        let (_, mut paths) = list_copy_from_paths(&req, &LocalFileAccess::Disabled)
1036            .await
1037            .unwrap();
1038        paths.sort();
1039        let mut expected = (0..16)
1040            .map(|i| format!("data.{i}.parquet"))
1041            .collect::<Vec<_>>();
1042        expected.sort();
1043        assert_eq!(paths, expected);
1044        assert_eq!(
1045            requests
1046                .lock()
1047                .unwrap()
1048                .iter()
1049                .filter(|(method, _)| *method == Method::GET)
1050                .count(),
1051            1
1052        );
1053
1054        for (name, kind) in [
1055            ("missing", object_store::ErrorKind::NotFound),
1056            ("denied", object_store::ErrorKind::PermissionDenied),
1057        ] {
1058            let mut req =
1059                copy_from_request(&format!("s3://bucket/backup/schema/1/{name}.parquet"), None);
1060            req.connection = connection.clone();
1061            assert!(matches!(
1062                list_copy_from_paths(&req, &LocalFileAccess::Disabled).await,
1063                Err(error::Error::ListObjects {
1064                    source: common_datasource::error::Error::ListObjects { error, .. }, ..
1065                }) if error.kind() == kind
1066            ));
1067        }
1068        server.abort();
1069    }
1070
1071    fn test_schema_matches(from: (DataType, bool), to: (DataType, bool), matches: bool) {
1072        let s1 = Arc::new(Schema::new(vec![Field::new("col", from.0.clone(), from.1)]));
1073        let s2 = Arc::new(Schema::new(vec![Field::new("col", to.0.clone(), to.1)]));
1074        let res = ensure_schema_compatible(&s1, &s2);
1075        assert_eq!(
1076            matches,
1077            res.is_ok(),
1078            "from data type: {}, to data type: {}, expected: {}, but got: {}",
1079            from.0,
1080            to.0,
1081            matches,
1082            res.is_ok()
1083        )
1084    }
1085
1086    #[test]
1087    fn test_ensure_datatype_matches_ignore_timezone() {
1088        test_schema_matches(
1089            (
1090                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
1091                true,
1092            ),
1093            (
1094                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
1095                true,
1096            ),
1097            true,
1098        );
1099
1100        test_schema_matches(
1101            (
1102                DataType::Timestamp(
1103                    datatypes::arrow::datatypes::TimeUnit::Second,
1104                    Some("UTC".into()),
1105                ),
1106                true,
1107            ),
1108            (
1109                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
1110                true,
1111            ),
1112            true,
1113        );
1114
1115        test_schema_matches(
1116            (
1117                DataType::Timestamp(
1118                    datatypes::arrow::datatypes::TimeUnit::Second,
1119                    Some("UTC".into()),
1120                ),
1121                true,
1122            ),
1123            (
1124                DataType::Timestamp(
1125                    datatypes::arrow::datatypes::TimeUnit::Second,
1126                    Some("PDT".into()),
1127                ),
1128                true,
1129            ),
1130            true,
1131        );
1132
1133        test_schema_matches(
1134            (
1135                DataType::Timestamp(
1136                    datatypes::arrow::datatypes::TimeUnit::Second,
1137                    Some("UTC".into()),
1138                ),
1139                true,
1140            ),
1141            (
1142                DataType::Timestamp(
1143                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
1144                    Some("UTC".into()),
1145                ),
1146                true,
1147            ),
1148            true,
1149        );
1150
1151        test_schema_matches((DataType::Int8, true), (DataType::Int8, true), true);
1152
1153        test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
1154    }
1155
1156    #[test]
1157    fn test_data_type_equals_ignore_timezone_with_options() {
1158        test_schema_matches(
1159            (
1160                DataType::Timestamp(
1161                    datatypes::arrow::datatypes::TimeUnit::Microsecond,
1162                    Some("UTC".into()),
1163                ),
1164                true,
1165            ),
1166            (
1167                DataType::Timestamp(
1168                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
1169                    Some("PDT".into()),
1170                ),
1171                true,
1172            ),
1173            true,
1174        );
1175
1176        test_schema_matches(
1177            (DataType::Utf8, true),
1178            (
1179                DataType::Timestamp(
1180                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
1181                    Some("PDT".into()),
1182                ),
1183                true,
1184            ),
1185            true,
1186        );
1187
1188        test_schema_matches(
1189            (
1190                DataType::Timestamp(
1191                    datatypes::arrow::datatypes::TimeUnit::Millisecond,
1192                    Some("PDT".into()),
1193                ),
1194                true,
1195            ),
1196            (DataType::Utf8, true),
1197            true,
1198        );
1199    }
1200
1201    #[test]
1202    fn test_map_to_binary_json_compatibility() {
1203        // Test Map -> Binary conversion for JSON types
1204        let map_type = DataType::Map(
1205            Arc::new(Field::new(
1206                "key_value",
1207                DataType::Struct(
1208                    vec![
1209                        Field::new("key", DataType::Utf8, false),
1210                        Field::new("value", DataType::Utf8, false),
1211                    ]
1212                    .into(),
1213                ),
1214                false,
1215            )),
1216            false,
1217        );
1218
1219        test_schema_matches((map_type, false), (DataType::Binary, true), true);
1220
1221        test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
1222        test_schema_matches((DataType::Utf8, true), (DataType::Binary, true), true);
1223    }
1224
1225    fn make_test_schema(v: &[Field]) -> Arc<Schema> {
1226        Arc::new(Schema::new(v.to_vec()))
1227    }
1228
1229    #[test]
1230    fn test_compatible_file_schema() {
1231        let file_schema0 = make_test_schema(&[
1232            Field::new("c1", DataType::UInt8, true),
1233            Field::new("c2", DataType::UInt8, true),
1234        ]);
1235
1236        let table_schema = make_test_schema(&[
1237            Field::new("c1", DataType::Int16, true),
1238            Field::new("c2", DataType::Int16, true),
1239            Field::new("c3", DataType::Int16, true),
1240        ]);
1241
1242        let compat_schema = make_test_schema(&[
1243            Field::new("c1", DataType::Int16, true),
1244            Field::new("c2", DataType::Int16, true),
1245        ]);
1246
1247        let (_, tp, _) =
1248            generated_schema_projection_and_compatible_file_schema(&file_schema0, &table_schema);
1249
1250        assert_eq!(table_schema.project(&tp).unwrap(), *compat_schema);
1251    }
1252
1253    #[test]
1254    fn test_schema_projection() {
1255        let file_schema0 = make_test_schema(&[
1256            Field::new("c1", DataType::UInt8, true),
1257            Field::new("c2", DataType::UInt8, true),
1258            Field::new("c3", DataType::UInt8, true),
1259        ]);
1260
1261        let file_schema1 = make_test_schema(&[
1262            Field::new("c3", DataType::UInt8, true),
1263            Field::new("c4", DataType::UInt8, true),
1264        ]);
1265
1266        let file_schema2 = make_test_schema(&[
1267            Field::new("c3", DataType::UInt8, true),
1268            Field::new("c4", DataType::UInt8, true),
1269            Field::new("c5", DataType::UInt8, true),
1270        ]);
1271
1272        let file_schema3 = make_test_schema(&[
1273            Field::new("c1", DataType::UInt8, true),
1274            Field::new("c2", DataType::UInt8, true),
1275        ]);
1276
1277        let table_schema = make_test_schema(&[
1278            Field::new("c3", DataType::UInt8, true),
1279            Field::new("c4", DataType::UInt8, true),
1280            Field::new("c5", DataType::UInt8, true),
1281        ]);
1282
1283        let tests = [
1284            (&file_schema0, &table_schema, true), // intersection
1285            (&file_schema1, &table_schema, true), // subset
1286            (&file_schema2, &table_schema, true), // full-eq
1287            (&file_schema3, &table_schema, true), // non-intersection
1288        ];
1289
1290        for test in tests {
1291            let (fp, tp, _) =
1292                generated_schema_projection_and_compatible_file_schema(test.0, test.1);
1293            assert_eq!(test.0.project(&fp).unwrap(), test.1.project(&tp).unwrap());
1294        }
1295    }
1296
1297    #[test]
1298    fn test_csv_reader_schema_for_skip_bad_records() {
1299        let file_schema = make_test_schema(&[
1300            Field::new("id", DataType::Utf8, true),
1301            Field::new("jsons", DataType::Utf8, true),
1302            Field::new("ts", DataType::Utf8, true),
1303        ]);
1304        let compat_schema = make_test_schema(&[
1305            Field::new("id", DataType::UInt32, true),
1306            Field::new("jsons", DataType::Binary, true),
1307            Field::new(
1308                "ts",
1309                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1310                true,
1311            ),
1312        ]);
1313
1314        let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1315
1316        assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1317        assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1318        assert_eq!(
1319            reader_schema.field(2).data_type(),
1320            compat_schema.field(2).data_type()
1321        );
1322    }
1323
1324    fn make_csv_metadata(schema: Arc<Schema>, has_header: bool) -> FileMetadata {
1325        FileMetadata::Csv {
1326            schema,
1327            format: CsvFormat {
1328                has_header,
1329                ..CsvFormat::default()
1330            },
1331            path: "test.csv".to_string(),
1332        }
1333    }
1334
1335    fn make_strict_csv_metadata(schema: Arc<Schema>) -> FileMetadata {
1336        FileMetadata::Csv {
1337            schema,
1338            format: CsvFormat {
1339                strict_headers: true,
1340                ..CsvFormat::default()
1341            },
1342            path: "test.csv".to_string(),
1343        }
1344    }
1345
1346    fn assert_field(schema: &Schema, idx: usize, name: &str, data_type: &DataType) {
1347        let field = schema.field(idx);
1348        assert_eq!(field.name(), name);
1349        assert_eq!(field.data_type(), data_type);
1350    }
1351
1352    #[test]
1353    fn test_strict_csv_headers_allows_reordered_columns() {
1354        let file_schema = make_test_schema(&[
1355            Field::new("ts", DataType::Utf8, true),
1356            Field::new("host_id", DataType::UInt8, true),
1357            Field::new("reading_value", DataType::Float64, true),
1358        ]);
1359        let table_schema = make_test_schema(&[
1360            Field::new("host_id", DataType::UInt32, true),
1361            Field::new("reading_value", DataType::Float64, true),
1362            Field::new("ts", DataType::Utf8, true),
1363        ]);
1364
1365        validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1366            .unwrap();
1367    }
1368
1369    #[test]
1370    fn test_strict_csv_headers_rejects_unknown_columns() {
1371        let file_schema = make_test_schema(&[
1372            Field::new("host_id", DataType::UInt8, true),
1373            Field::new("reading_value", DataType::Float64, true),
1374            Field::new("ts", DataType::Utf8, true),
1375            Field::new("extra", DataType::Utf8, true),
1376        ]);
1377        let table_schema = make_test_schema(&[
1378            Field::new("host_id", DataType::UInt32, true),
1379            Field::new("reading_value", DataType::Float64, true),
1380            Field::new("ts", DataType::Utf8, true),
1381        ]);
1382
1383        let err =
1384            validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1385                .unwrap_err();
1386
1387        assert!(matches!(
1388            err,
1389            error::Error::CsvHeaderMismatch {
1390                unknown_columns,
1391                missing_columns,
1392                duplicate_columns,
1393                ..
1394            } if unknown_columns == vec!["extra".to_string()]
1395                && missing_columns.is_empty()
1396                && duplicate_columns.is_empty()
1397        ));
1398    }
1399
1400    #[test]
1401    fn test_strict_csv_headers_rejects_missing_columns() {
1402        let file_schema = make_test_schema(&[
1403            Field::new("host_id", DataType::UInt8, true),
1404            Field::new("ts", DataType::Utf8, true),
1405        ]);
1406        let table_schema = make_test_schema(&[
1407            Field::new("host_id", DataType::UInt32, true),
1408            Field::new("reading_value", DataType::Float64, true),
1409            Field::new("ts", DataType::Utf8, true),
1410        ]);
1411
1412        let err =
1413            validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1414                .unwrap_err();
1415
1416        assert!(matches!(
1417            err,
1418            error::Error::CsvHeaderMismatch {
1419                unknown_columns,
1420                missing_columns,
1421                duplicate_columns,
1422                ..
1423            } if unknown_columns.is_empty()
1424                && missing_columns == vec!["reading_value".to_string()]
1425                && duplicate_columns.is_empty()
1426        ));
1427    }
1428
1429    #[test]
1430    fn test_strict_csv_headers_rejects_duplicate_columns() {
1431        let file_schema = make_test_schema(&[
1432            Field::new("host_id", DataType::UInt8, true),
1433            Field::new("reading_value", DataType::Float64, true),
1434            Field::new("ts", DataType::Utf8, true),
1435            Field::new("host_id", DataType::UInt16, true),
1436        ]);
1437        let table_schema = make_test_schema(&[
1438            Field::new("host_id", DataType::UInt32, true),
1439            Field::new("reading_value", DataType::Float64, true),
1440            Field::new("ts", DataType::Utf8, true),
1441        ]);
1442
1443        let err =
1444            validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1445                .unwrap_err();
1446
1447        assert!(matches!(
1448            err,
1449            error::Error::CsvHeaderMismatch {
1450                unknown_columns,
1451                missing_columns,
1452                duplicate_columns,
1453                ..
1454            } if unknown_columns.is_empty()
1455                && missing_columns.is_empty()
1456                && duplicate_columns == vec!["host_id".to_string()]
1457        ));
1458    }
1459
1460    #[test]
1461    fn test_headerless_csv_schema_projection_is_positional() {
1462        let file_schema = make_test_schema(&[
1463            Field::new("column_1", DataType::UInt8, true),
1464            Field::new("column_2", DataType::Float64, true),
1465            Field::new("column_3", DataType::Utf8, true),
1466        ]);
1467        let table_schema = make_test_schema(&[
1468            Field::new("host_id", DataType::UInt32, true),
1469            Field::new("reading_value", DataType::Float64, true),
1470            Field::new(
1471                "ts",
1472                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1473                true,
1474            ),
1475        ]);
1476
1477        let mapping =
1478            copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1479
1480        assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1481        assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1482        assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1483        assert_field(
1484            &mapping.compat_file_schema,
1485            1,
1486            "reading_value",
1487            &DataType::Float64,
1488        );
1489        assert_field(
1490            &mapping.compat_file_schema,
1491            2,
1492            "ts",
1493            table_schema.field(2).data_type(),
1494        );
1495        assert_eq!(
1496            mapping
1497                .compat_file_schema
1498                .project(&mapping.file_projection)
1499                .unwrap(),
1500            table_schema.project(&mapping.table_projection).unwrap()
1501        );
1502    }
1503
1504    #[test]
1505    fn test_headerless_csv_schema_projection_ignores_extra_file_columns() {
1506        let file_schema = make_test_schema(&[
1507            Field::new("column_1", DataType::UInt8, true),
1508            Field::new("column_2", DataType::Float64, true),
1509            Field::new("column_3", DataType::Utf8, true),
1510            Field::new("column_4", DataType::Utf8, true),
1511        ]);
1512        let table_schema = make_test_schema(&[
1513            Field::new("host_id", DataType::UInt32, true),
1514            Field::new("reading_value", DataType::Float64, true),
1515            Field::new("ts", DataType::Utf8, true),
1516        ]);
1517
1518        let mapping =
1519            copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1520
1521        assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1522        assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1523        assert_eq!(mapping.compat_file_schema.fields().len(), 4);
1524        assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1525        assert_field(
1526            &mapping.compat_file_schema,
1527            1,
1528            "reading_value",
1529            &DataType::Float64,
1530        );
1531        assert_field(&mapping.compat_file_schema, 2, "ts", &DataType::Utf8);
1532        assert_field(&mapping.compat_file_schema, 3, "column_4", &DataType::Utf8);
1533    }
1534
1535    #[test]
1536    fn test_headerless_csv_schema_projection_supports_prefix_import() {
1537        let file_schema = make_test_schema(&[
1538            Field::new("column_1", DataType::UInt8, true),
1539            Field::new("column_2", DataType::Float64, true),
1540        ]);
1541        let table_schema = make_test_schema(&[
1542            Field::new("host_id", DataType::UInt32, true),
1543            Field::new("reading_value", DataType::Float64, true),
1544            Field::new("ts", DataType::Utf8, true),
1545        ]);
1546
1547        let mapping =
1548            copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1549
1550        assert_eq!(mapping.file_projection, vec![0, 1]);
1551        assert_eq!(mapping.table_projection, vec![0, 1]);
1552        assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1553        assert_field(
1554            &mapping.compat_file_schema,
1555            1,
1556            "reading_value",
1557            &DataType::Float64,
1558        );
1559        assert_eq!(
1560            mapping
1561                .compat_file_schema
1562                .project(&mapping.file_projection)
1563                .unwrap(),
1564            table_schema.project(&mapping.table_projection).unwrap()
1565        );
1566    }
1567
1568    #[test]
1569    fn test_csv_reader_schema_for_skip_bad_records_uses_positional_mapping() {
1570        let file_schema = make_test_schema(&[
1571            Field::new("column_1", DataType::Utf8, true),
1572            Field::new("column_2", DataType::Utf8, true),
1573            Field::new("column_3", DataType::Utf8, true),
1574        ]);
1575        let table_schema = make_test_schema(&[
1576            Field::new("host_id", DataType::UInt32, true),
1577            Field::new("jsons", DataType::Binary, true),
1578            Field::new(
1579                "ts",
1580                DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1581                true,
1582            ),
1583        ]);
1584        let mapping = copy_from_schema_mapping(
1585            &make_csv_metadata(file_schema.clone(), false),
1586            &table_schema,
1587        );
1588        let compat_schema = Arc::new(mapping.compat_file_schema);
1589
1590        let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1591
1592        assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1593        assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1594        assert_eq!(
1595            reader_schema.field(2).data_type(),
1596            table_schema.field(2).data_type()
1597        );
1598    }
1599}