1use std::collections::{BTreeSet, HashMap, HashSet};
16use std::future::Future;
17use std::pin::Pin;
18use std::sync::Arc;
19use std::task::{Context, Poll};
20
21use client::{Output, OutputData, OutputMeta};
22use common_base::readable_size::ReadableSize;
23use common_datasource::file_format::csv::{
24 CsvFormat, is_skippable_arrow_error, tolerant_csv_stream,
25};
26use common_datasource::file_format::json::JsonFormat;
27use common_datasource::file_format::orc::{ReaderAdapter, infer_orc_schema, new_orc_stream_reader};
28use common_datasource::file_format::{FileFormat, Format, file_to_stream};
29use common_datasource::lister::{Lister, Source};
30use common_datasource::object_store::{LocalFileAccess, build_backend_with_path};
31use common_query::{OutputCost, OutputRows};
32use common_recordbatch::DfSendableRecordBatchStream;
33use common_recordbatch::adapter::RecordBatchStreamTypeAdapter;
34use common_telemetry::{debug, tracing};
35use datafusion::datasource::physical_plan::{CsvSource, FileSource, JsonSource};
36use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
37use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata;
38use datafusion_common::DataFusionError;
39use datafusion_common::arrow::error::ArrowError;
40use datafusion_common::config::CsvOptions;
41use datafusion_expr::Expr;
42use datatypes::arrow::compute::can_cast_types;
43use datatypes::arrow::datatypes::{DataType as ArrowDataType, Schema, SchemaRef};
44use datatypes::arrow::record_batch::RecordBatch;
45use datatypes::vectors::Helper;
46use futures_util::StreamExt;
47use object_store::{EntryMode, ObjectStore};
48use regex::Regex;
49use session::context::QueryContextRef;
50use snafu::{ResultExt, ensure};
51use table::requests::{CopyTableRequest, InsertRequest};
52use table::table_reference::TableReference;
53use tokio_util::compat::FuturesAsyncReadCompatExt;
54
55use crate::error::{self, IntoVectorsSnafu, Result};
56use crate::statement::StatementExecutor;
57
58const DEFAULT_BATCH_SIZE: usize = 8192;
59const DEFAULT_READ_BUFFER: usize = 256 * 1024;
60
61enum FileMetadata {
62 Parquet {
63 schema: SchemaRef,
64 metadata: ArrowReaderMetadata,
65 path: String,
66 },
67 Orc {
68 schema: SchemaRef,
69 path: String,
70 },
71 Json {
72 schema: SchemaRef,
73 format: JsonFormat,
74 path: String,
75 },
76 Csv {
77 schema: SchemaRef,
78 format: CsvFormat,
79 path: String,
80 },
81}
82
83impl FileMetadata {
84 pub fn schema(&self) -> &SchemaRef {
86 match self {
87 FileMetadata::Parquet { schema, .. } => schema,
88 FileMetadata::Orc { schema, .. } => schema,
89 FileMetadata::Json { schema, .. } => schema,
90 FileMetadata::Csv { schema, .. } => schema,
91 }
92 }
93}
94
95async fn list_copy_from_paths(
96 req: &CopyTableRequest,
97 local_file_access: &LocalFileAccess,
98) -> Result<(ObjectStore, Vec<String>)> {
99 let backend = build_backend_with_path(&req.location, &req.connection, local_file_access)
100 .await
101 .context(error::BuildBackendSnafu)?;
102 let regex = req
103 .pattern
104 .as_ref()
105 .map(|x| Regex::new(x))
106 .transpose()
107 .context(error::BuildRegexSnafu)?;
108
109 if let Some(filename) = &backend.object_path {
112 let is_file = backend
113 .is_file(filename)
114 .await
115 .with_context(|_| common_datasource::error::ListObjectsSnafu {
116 path: req.location.clone(),
117 })
118 .context(error::ListObjectsSnafu)?;
119 let paths = if is_file {
120 vec![filename.clone()]
121 } else {
122 vec![]
123 };
124 return Ok((backend.object_store, paths));
125 }
126
127 let lister = Lister::new(
128 backend.object_store.clone(),
129 Source::Dir,
130 req.location.clone(),
131 regex,
132 );
133
134 let entries = lister.list().await.context(error::ListObjectsSnafu)?;
135 debug!(
136 "Copy from location: {:?}, entries: {entries:?}",
137 req.location
138 );
139 let paths = entries
140 .into_iter()
141 .filter(|entry| entry.metadata().mode() == EntryMode::FILE)
142 .map(|entry| entry.path().to_string())
143 .collect();
144 Ok((backend.object_store, paths))
145}
146
147impl StatementExecutor {
148 async fn collect_metadata(
149 &self,
150 object_store: &ObjectStore,
151 format: Format,
152 path: String,
153 ) -> Result<FileMetadata> {
154 match format {
155 Format::Csv(format) => Ok(FileMetadata::Csv {
156 schema: Arc::new(
157 format
158 .infer_schema(object_store, &path)
159 .await
160 .context(error::InferSchemaSnafu { path: &path })?,
161 ),
162 format,
163 path,
164 }),
165 Format::Json(format) => Ok(FileMetadata::Json {
166 schema: Arc::new(
167 format
168 .infer_schema(object_store, &path)
169 .await
170 .context(error::InferSchemaSnafu { path: &path })?,
171 ),
172 format,
173 path,
174 }),
175 Format::Parquet(_) => {
176 let meta = object_store
177 .stat(&path)
178 .await
179 .context(error::ReadObjectSnafu { path: &path })?;
180 let mut reader = object_store
181 .reader(&path)
182 .await
183 .context(error::ReadObjectSnafu { path: &path })?
184 .into_futures_async_read(0..meta.content_length())
185 .await
186 .context(error::ReadObjectSnafu { path: &path })?
187 .compat();
188 let metadata = ArrowReaderMetadata::load_async(&mut reader, Default::default())
189 .await
190 .context(error::ReadParquetMetadataSnafu)?;
191
192 Ok(FileMetadata::Parquet {
193 schema: metadata.schema().clone(),
194 metadata,
195 path,
196 })
197 }
198 Format::Orc(_) => {
199 let meta = object_store
200 .stat(&path)
201 .await
202 .context(error::ReadObjectSnafu { path: &path })?;
203
204 let reader = object_store
205 .reader(&path)
206 .await
207 .context(error::ReadObjectSnafu { path: &path })?;
208
209 let schema = infer_orc_schema(ReaderAdapter::new(reader, meta.content_length()))
210 .await
211 .context(error::ReadOrcSnafu)?;
212
213 Ok(FileMetadata::Orc {
214 schema: Arc::new(schema),
215 path,
216 })
217 }
218 }
219 }
220
221 async fn build_read_stream(
222 &self,
223 compat_schema: SchemaRef,
224 object_store: &ObjectStore,
225 file_metadata: &FileMetadata,
226 projection: Vec<usize>,
227 filters: Vec<Expr>,
228 ) -> Result<DfSendableRecordBatchStream> {
229 match file_metadata {
230 FileMetadata::Csv {
231 format,
232 path,
233 schema,
234 } => {
235 let output_schema = Arc::new(
236 compat_schema
237 .project(&projection)
238 .context(error::ProjectSchemaSnafu)?,
239 );
240
241 let options = CsvOptions::default()
242 .with_has_header(format.has_header)
243 .with_delimiter(format.delimiter);
244 let csv_source = CsvSource::new(schema.clone())
245 .with_csv_options(options)
246 .with_batch_size(DEFAULT_BATCH_SIZE);
247 let stream = if format.skip_bad_records {
248 let reader_schema =
249 csv_reader_schema_for_skip_bad_records(schema, &compat_schema);
250 tolerant_csv_stream(
251 object_store,
252 path,
253 Arc::new(reader_schema),
254 projection.clone(),
255 format,
256 )
257 .await
258 .context(error::BuildFileStreamSnafu)?
259 } else {
260 file_to_stream(
261 object_store,
262 path,
263 csv_source,
264 Some(projection),
265 format.compression_type,
266 )
267 .await
268 .context(error::BuildFileStreamSnafu)?
269 };
270
271 let stream = Box::pin(
272 RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
275 .with_filter(filters)
276 .context(error::PhysicalExprSnafu)?,
277 );
278 if format.skip_bad_records {
279 Ok(Box::pin(SkipBadRecordsStream::new(stream, path)))
280 } else {
281 Ok(stream)
282 }
283 }
284 FileMetadata::Json {
285 path,
286 format,
287 schema,
288 } => {
289 let output_schema = Arc::new(
290 compat_schema
291 .project(&projection)
292 .context(error::ProjectSchemaSnafu)?,
293 );
294
295 let json_source =
296 JsonSource::new(schema.clone()).with_batch_size(DEFAULT_BATCH_SIZE);
297 let stream = file_to_stream(
298 object_store,
299 path,
300 json_source,
301 Some(projection),
302 format.compression_type,
303 )
304 .await
305 .context(error::BuildFileStreamSnafu)?;
306
307 Ok(Box::pin(
308 RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
311 .with_filter(filters)
312 .context(error::PhysicalExprSnafu)?,
313 ))
314 }
315 FileMetadata::Parquet { metadata, path, .. } => {
316 let meta = object_store
317 .stat(path)
318 .await
319 .context(error::ReadObjectSnafu { path })?;
320 let reader = object_store
321 .reader_with(path)
322 .chunk(DEFAULT_READ_BUFFER)
323 .await
324 .context(error::ReadObjectSnafu { path })?
325 .into_futures_async_read(0..meta.content_length())
326 .await
327 .context(error::ReadObjectSnafu { path })?
328 .compat();
329 let builder =
330 ParquetRecordBatchStreamBuilder::new_with_metadata(reader, metadata.clone());
331 let stream = builder
332 .build()
333 .context(error::BuildParquetRecordBatchStreamSnafu)?;
334
335 let output_schema = Arc::new(
336 compat_schema
337 .project(&projection)
338 .context(error::ProjectSchemaSnafu)?,
339 );
340 Ok(Box::pin(
341 RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
342 .with_filter(filters)
343 .context(error::PhysicalExprSnafu)?,
344 ))
345 }
346 FileMetadata::Orc { path, .. } => {
347 let meta = object_store
348 .stat(path)
349 .await
350 .context(error::ReadObjectSnafu { path })?;
351
352 let reader = object_store
353 .reader_with(path)
354 .chunk(DEFAULT_READ_BUFFER)
355 .await
356 .context(error::ReadObjectSnafu { path })?;
357 let stream =
358 new_orc_stream_reader(ReaderAdapter::new(reader, meta.content_length()))
359 .await
360 .context(error::ReadOrcSnafu)?;
361
362 let output_schema = Arc::new(
363 compat_schema
364 .project(&projection)
365 .context(error::ProjectSchemaSnafu)?,
366 );
367
368 Ok(Box::pin(
369 RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
370 .with_filter(filters)
371 .context(error::PhysicalExprSnafu)?,
372 ))
373 }
374 }
375 }
376
377 #[tracing::instrument(skip_all)]
378 pub async fn copy_table_from(
379 &self,
380 req: CopyTableRequest,
381 query_ctx: QueryContextRef,
382 ) -> Result<Output> {
383 let table_ref = TableReference {
384 catalog: &req.catalog_name,
385 schema: &req.schema_name,
386 table: &req.table_name,
387 };
388 let table = self.get_table(&table_ref).await?;
389 let format = Format::try_from(&req.with).context(error::ParseFileFormatSnafu)?;
390 let (object_store, paths) = list_copy_from_paths(&req, &self.local_file_access).await?;
391 let mut files = Vec::with_capacity(paths.len());
392 let table_schema = table.schema().arrow_schema().clone();
393 let filters = table
394 .schema()
395 .timestamp_column()
396 .and_then(|c| {
397 common_query::logical_plan::build_same_type_ts_filter(c, req.timestamp_range)
398 })
399 .into_iter()
400 .collect::<Vec<_>>();
401
402 for path in paths {
403 let file_metadata = self
404 .collect_metadata(&object_store, format.clone(), path)
405 .await?;
406
407 validate_csv_headers_if_required(&file_metadata, &table_schema)?;
408 let schema_mapping = copy_from_schema_mapping(&file_metadata, &table_schema);
409 let projected_file_schema = Arc::new(
410 file_metadata
411 .schema()
412 .project(&schema_mapping.file_projection)
413 .context(error::ProjectSchemaSnafu)?,
414 );
415 let projected_table_schema = Arc::new(
416 table_schema
417 .project(&schema_mapping.table_projection)
418 .context(error::ProjectSchemaSnafu)?,
419 );
420 ensure_schema_compatible(&projected_file_schema, &projected_table_schema)?;
421
422 files.push((
423 Arc::new(schema_mapping.compat_file_schema),
424 schema_mapping.file_projection,
425 projected_table_schema,
426 file_metadata,
427 ))
428 }
429
430 let mut rows_inserted = 0;
431 let mut insert_cost = 0;
432 let max_insert_rows = req
433 .limit
434 .map(|n| {
435 usize::try_from(n).map_err(|_| {
436 error::InvalidCopyParameterSnafu {
437 key: "limit".to_string(),
438 value: n.to_string(),
439 }
440 .build()
441 })
442 })
443 .transpose()?;
444 if max_insert_rows == Some(0) {
445 return Ok(gen_insert_output(rows_inserted, insert_cost));
446 }
447
448 let mut accepted_rows = 0;
449 for (compat_schema, file_schema_projection, projected_table_schema, file_metadata) in files
450 {
451 let mut stream = self
452 .build_read_stream(
453 compat_schema,
454 &object_store,
455 &file_metadata,
456 file_schema_projection,
457 filters.clone(),
458 )
459 .await?;
460
461 let fields = projected_table_schema
462 .fields()
463 .iter()
464 .map(|f| f.name().clone())
465 .collect::<Vec<_>>();
466
467 let pending_mem_threshold = ReadableSize::mb(32).as_bytes();
469 let mut pending_mem_size = 0;
470 let mut pending = vec![];
471
472 while let Some(r) = stream.next().await {
473 let record_batch = r.context(error::ReadDfRecordBatchSnafu)?;
474 let record_batch = if let Some(max_insert_rows) = max_insert_rows {
475 let remaining_rows = max_insert_rows - accepted_rows;
476 if record_batch.num_rows() > remaining_rows {
477 record_batch.slice(0, remaining_rows)
478 } else {
479 record_batch
480 }
481 } else {
482 record_batch
483 };
484 let record_batch_rows = record_batch.num_rows();
485 let vectors =
486 Helper::try_into_vectors(record_batch.columns()).context(IntoVectorsSnafu)?;
487
488 pending_mem_size += vectors.iter().map(|v| v.memory_size()).sum::<usize>();
489
490 let columns_values = fields
491 .iter()
492 .cloned()
493 .zip(vectors)
494 .collect::<HashMap<_, _>>();
495
496 pending.push(self.inserter.handle_table_insert(
497 InsertRequest {
498 catalog_name: req.catalog_name.clone(),
499 schema_name: req.schema_name.clone(),
500 table_name: req.table_name.clone(),
501 columns_values,
502 skip_wal: query_ctx.skip_wal(),
503 },
504 query_ctx.clone(),
505 ));
506 accepted_rows += record_batch_rows;
507
508 if pending_mem_size as u64 >= pending_mem_threshold {
509 let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
510 rows_inserted += rows;
511 insert_cost += cost;
512 }
513
514 if let Some(max_insert_rows) = max_insert_rows
515 && accepted_rows == max_insert_rows
516 {
517 if !pending.is_empty() {
518 let (rows, cost) =
519 batch_insert(&mut pending, &mut pending_mem_size).await?;
520 rows_inserted += rows;
521 insert_cost += cost;
522 }
523 return Ok(gen_insert_output(rows_inserted, insert_cost));
524 }
525 }
526
527 if !pending.is_empty() {
528 let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
529 rows_inserted += rows;
530 insert_cost += cost;
531 }
532 }
533
534 Ok(gen_insert_output(rows_inserted, insert_cost))
535 }
536}
537
538fn gen_insert_output(rows_inserted: usize, insert_cost: usize) -> Output {
539 Output::new(
540 OutputData::AffectedRows(rows_inserted),
541 OutputMeta::new_with_cost(insert_cost),
542 )
543}
544
545struct SkipBadRecordsStream {
546 inner: DfSendableRecordBatchStream,
547 path: String,
548}
549
550impl SkipBadRecordsStream {
551 fn new(inner: DfSendableRecordBatchStream, path: impl Into<String>) -> Self {
552 Self {
553 inner,
554 path: path.into(),
555 }
556 }
557}
558
559impl datafusion::physical_plan::RecordBatchStream for SkipBadRecordsStream {
560 fn schema(&self) -> SchemaRef {
561 self.inner.schema()
562 }
563}
564
565impl futures::Stream for SkipBadRecordsStream {
566 type Item = datafusion_common::Result<RecordBatch>;
567
568 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
569 let this = self.get_mut();
570 loop {
571 match this.inner.as_mut().poll_next(cx) {
572 Poll::Ready(Some(Err(error))) if is_skippable_record_error(&error) => {
573 common_telemetry::warn!(
574 "Skipping bad record while copying from {}: {}",
575 this.path,
576 error
577 );
578 continue;
579 }
580 other => return other,
581 }
582 }
583 }
584}
585
586fn is_skippable_record_error(error: &DataFusionError) -> bool {
587 match error {
588 DataFusionError::ArrowError(error, _) => is_skippable_arrow_error(error),
589 DataFusionError::External(error) => error
590 .downcast_ref::<ArrowError>()
591 .is_some_and(is_skippable_arrow_error),
592 DataFusionError::Context(_, error) => is_skippable_record_error(error),
593 _ => false,
594 }
595}
596
597async fn batch_insert(
599 pending: &mut Vec<impl Future<Output = Result<Output>>>,
600 pending_bytes: &mut usize,
601) -> Result<(OutputRows, OutputCost)> {
602 let batch = pending.drain(..);
603 let result = futures::future::try_join_all(batch)
604 .await?
605 .iter()
606 .map(|o| o.extract_rows_and_cost())
607 .reduce(|(a, b), (c, d)| (a + c, b + d))
608 .unwrap_or((0, 0));
609 *pending_bytes = 0;
610 Ok(result)
611}
612
613fn can_cast_types_for_greptime(from: &ArrowDataType, to: &ArrowDataType) -> bool {
615 if let ArrowDataType::Map(_, _) = from
617 && let ArrowDataType::Binary = to
618 {
619 return true;
620 }
621
622 can_cast_types(from, to)
624}
625
626fn csv_reader_schema_for_skip_bad_records(file: &SchemaRef, compat: &SchemaRef) -> Schema {
627 let fields = file
628 .fields()
629 .iter()
630 .enumerate()
631 .map(|(idx, file_field)| match compat.fields().get(idx) {
632 Some(compat_field) if can_csv_reader_parse_type(compat_field.data_type()) => {
633 compat_field.clone()
634 }
635 _ => file_field.clone(),
636 })
637 .collect::<Vec<_>>();
638
639 Schema::new_with_metadata(fields, file.metadata().clone())
640}
641
642fn can_csv_reader_parse_type(data_type: &ArrowDataType) -> bool {
643 match data_type {
644 ArrowDataType::Boolean
645 | ArrowDataType::Decimal32(_, _)
646 | ArrowDataType::Decimal64(_, _)
647 | ArrowDataType::Decimal128(_, _)
648 | ArrowDataType::Decimal256(_, _)
649 | ArrowDataType::Int8
650 | ArrowDataType::Int16
651 | ArrowDataType::Int32
652 | ArrowDataType::Int64
653 | ArrowDataType::UInt8
654 | ArrowDataType::UInt16
655 | ArrowDataType::UInt32
656 | ArrowDataType::UInt64
657 | ArrowDataType::Float32
658 | ArrowDataType::Float64
659 | ArrowDataType::Date32
660 | ArrowDataType::Date64
661 | ArrowDataType::Time32(_)
662 | ArrowDataType::Time64(_)
663 | ArrowDataType::Timestamp(_, _)
664 | ArrowDataType::Null
665 | ArrowDataType::Utf8
666 | ArrowDataType::Utf8View => true,
667 ArrowDataType::Dictionary(_, value_type) => value_type.as_ref() == &ArrowDataType::Utf8,
668 _ => false,
669 }
670}
671
672fn ensure_schema_compatible(from: &SchemaRef, to: &SchemaRef) -> Result<()> {
673 let not_match = from
674 .fields
675 .iter()
676 .zip(to.fields.iter())
677 .map(|(l, r)| (l.data_type(), r.data_type()))
678 .enumerate()
679 .find(|(_, (l, r))| !can_cast_types_for_greptime(l, r));
680
681 if let Some((index, _)) = not_match {
682 error::InvalidSchemaSnafu {
683 index,
684 table_schema: to.to_string(),
685 file_schema: from.to_string(),
686 }
687 .fail()
688 } else {
689 Ok(())
690 }
691}
692
693fn validate_csv_headers_if_required(file_metadata: &FileMetadata, table: &SchemaRef) -> Result<()> {
694 let FileMetadata::Csv {
695 schema,
696 format,
697 path,
698 } = file_metadata
699 else {
700 return Ok(());
701 };
702
703 if !format.strict_headers {
704 return Ok(());
705 }
706
707 let mut seen_file_columns = HashSet::with_capacity(schema.fields().len());
708 let duplicate_columns = schema
709 .fields()
710 .iter()
711 .filter_map(|field| {
712 if seen_file_columns.insert(field.name().clone()) {
713 None
714 } else {
715 Some(field.name().clone())
716 }
717 })
718 .collect::<BTreeSet<_>>()
719 .into_iter()
720 .collect::<Vec<_>>();
721 let file_columns = seen_file_columns.into_iter().collect::<BTreeSet<_>>();
722 let table_columns = table
723 .fields()
724 .iter()
725 .map(|field| field.name().clone())
726 .collect::<BTreeSet<_>>();
727 let unknown_columns = file_columns
728 .difference(&table_columns)
729 .cloned()
730 .collect::<Vec<_>>();
731 let missing_columns = table_columns
732 .difference(&file_columns)
733 .cloned()
734 .collect::<Vec<_>>();
735
736 ensure!(
737 unknown_columns.is_empty() && missing_columns.is_empty() && duplicate_columns.is_empty(),
738 error::CsvHeaderMismatchSnafu {
739 path,
740 unknown_columns,
741 missing_columns,
742 duplicate_columns,
743 }
744 );
745
746 Ok(())
747}
748
749fn generated_schema_projection_and_compatible_file_schema(
754 file: &SchemaRef,
755 table: &SchemaRef,
756) -> (Vec<usize>, Vec<usize>, Schema) {
757 let mut file_projection = Vec::with_capacity(file.fields.len());
758 let mut table_projection = Vec::with_capacity(file.fields.len());
759 let mut compatible_fields = file.fields.iter().cloned().collect::<Vec<_>>();
760 for (file_idx, file_field) in file.fields.iter().enumerate() {
761 if let Some((table_idx, table_field)) = table.fields.find(file_field.name()) {
762 file_projection.push(file_idx);
763 table_projection.push(table_idx);
764
765 compatible_fields[file_idx] = table_field.clone();
767 }
768 }
769
770 (
771 file_projection,
772 table_projection,
773 Schema::new(compatible_fields),
774 )
775}
776
777struct CopyFromSchemaMapping {
778 file_projection: Vec<usize>,
779 table_projection: Vec<usize>,
780 compat_file_schema: Schema,
781}
782
783fn copy_from_schema_mapping(
784 file_metadata: &FileMetadata,
785 table: &SchemaRef,
786) -> CopyFromSchemaMapping {
787 match file_metadata {
788 FileMetadata::Csv { schema, format, .. } if !format.has_header => {
789 generated_positional_schema_projection_and_compatible_file_schema(schema, table)
790 }
791 _ => {
792 let (file_projection, table_projection, compat_file_schema) =
793 generated_schema_projection_and_compatible_file_schema(
794 file_metadata.schema(),
795 table,
796 );
797 CopyFromSchemaMapping {
798 file_projection,
799 table_projection,
800 compat_file_schema,
801 }
802 }
803 }
804}
805
806fn generated_positional_schema_projection_and_compatible_file_schema(
807 file: &SchemaRef,
808 table: &SchemaRef,
809) -> CopyFromSchemaMapping {
810 let len = file.fields.len().min(table.fields.len());
811 let file_projection = (0..len).collect::<Vec<_>>();
812 let table_projection = (0..len).collect::<Vec<_>>();
813 let compatible_fields = file
814 .fields
815 .iter()
816 .enumerate()
817 .map(|(idx, file_field)| {
818 if idx < len {
819 table.fields[idx].clone()
820 } else {
821 file_field.clone()
822 }
823 })
824 .collect::<Vec<_>>();
825
826 CopyFromSchemaMapping {
827 file_projection,
828 table_projection,
829 compat_file_schema: Schema::new(compatible_fields),
830 }
831}
832
833#[cfg(test)]
834mod tests {
835 use std::sync::Arc;
836
837 use datatypes::arrow::datatypes::{DataType, Field, Schema};
838
839 use super::*;
840
841 fn copy_from_request(location: &str, pattern: Option<&str>) -> CopyTableRequest {
842 CopyTableRequest {
843 catalog_name: "greptime".into(),
844 schema_name: "public".into(),
845 table_name: "test".into(),
846 location: location.into(),
847 with: HashMap::new(),
848 connection: HashMap::new(),
849 pattern: pattern.map(String::from),
850 direction: table::requests::CopyDirection::Import,
851 timestamp_range: None,
852 limit: None,
853 }
854 }
855
856 #[tokio::test]
857 async fn test_copy_from_local_paths() {
858 use common_test_util::temp_dir::create_temp_dir;
859
860 let dir = create_temp_dir("copy_from_paths");
861 let nested = dir.path().join("nested");
862 std::fs::create_dir_all(nested.join("data.dir")).unwrap();
863 std::fs::write(nested.join("data file.parquet"), b"data").unwrap();
864 std::fs::write(nested.join("other.csv"), b"other").unwrap();
865 let access = LocalFileAccess::sandboxed(dir.path()).unwrap();
866 for location in [
867 "nested/data file.parquet".to_string(),
868 url::Url::from_file_path(nested.join("data file.parquet"))
869 .unwrap()
870 .to_string(),
871 ] {
872 let req = copy_from_request(&location, Some("does-not-match"));
874 let (store, paths) = list_copy_from_paths(&req, &access).await.unwrap();
875 assert_eq!(paths, ["data file.parquet"]);
876 assert_eq!(store.read(&paths[0]).await.unwrap().to_vec(), b"data");
877 }
878
879 let req = copy_from_request("nested/", Some("^data"));
880 let (_, paths) = list_copy_from_paths(&req, &access).await.unwrap();
881 assert_eq!(paths, ["data file.parquet"]);
882 let req = copy_from_request("nested/", None);
883 let (_, mut paths) = list_copy_from_paths(&req, &access).await.unwrap();
884 paths.sort();
885 assert_eq!(paths, ["data file.parquet", "other.csv"]);
886 let req = copy_from_request("nested/data.dir", None);
887 assert!(
888 list_copy_from_paths(&req, &access)
889 .await
890 .unwrap()
891 .1
892 .is_empty()
893 );
894 let req = copy_from_request("nested/", Some("does-not-match"));
895 assert!(
896 list_copy_from_paths(&req, &access)
897 .await
898 .unwrap()
899 .1
900 .is_empty()
901 );
902
903 let req = copy_from_request("nested/missing.parquet", None);
904 assert!(matches!(
905 list_copy_from_paths(&req, &access).await,
906 Err(error::Error::ListObjects {
907 source: common_datasource::error::Error::ListObjects { error, .. }, ..
908 }) if error.kind() == object_store::ErrorKind::NotFound
909 ));
910 let req = copy_from_request("nested/data file.parquet", Some("["));
911 assert!(matches!(
912 list_copy_from_paths(&req, &access).await,
913 Err(error::Error::BuildRegex { .. })
914 ));
915 let req = copy_from_request("nested/data file.parquet", None);
916 assert!(matches!(
917 list_copy_from_paths(&req, &LocalFileAccess::Disabled).await,
918 Err(error::Error::BuildBackend {
919 source: common_datasource::error::Error::LocalFileAccessDisabled { .. },
920 ..
921 })
922 ));
923 let req = copy_from_request("../outside.parquet", None);
924 assert!(matches!(
925 list_copy_from_paths(&req, &access).await,
926 Err(error::Error::BuildBackend {
927 source: common_datasource::error::Error::LocalFileAccessDenied { .. },
928 ..
929 })
930 ));
931 }
932
933 #[cfg(unix)]
934 #[tokio::test]
935 async fn test_copy_from_local_symlinks() {
936 use common_test_util::temp_dir::create_temp_dir;
937
938 let dir = create_temp_dir("copy_from_symlinks");
939 std::fs::write(dir.path().join("data.parquet"), b"data").unwrap();
940 std::os::unix::fs::symlink("data.parquet", dir.path().join("link.parquet")).unwrap();
941 std::os::unix::fs::symlink("missing.parquet", dir.path().join("dangling.parquet")).unwrap();
942 let access = LocalFileAccess::sandboxed(dir.path()).unwrap();
943
944 let req = copy_from_request("link.parquet", None);
945 assert!(
946 list_copy_from_paths(&req, &access)
947 .await
948 .unwrap()
949 .1
950 .is_empty()
951 );
952 let req = copy_from_request("./", None);
953 let (_, paths) = list_copy_from_paths(&req, &access).await.unwrap();
954 assert_eq!(paths, ["data.parquet"]);
955 let req = copy_from_request("dangling.parquet", None);
956 assert!(matches!(
957 list_copy_from_paths(&req, &access).await,
958 Err(error::Error::ListObjects {
959 source: common_datasource::error::Error::ListObjects { error, .. }, ..
960 }) if error.kind() == object_store::ErrorKind::NotFound
961 ));
962 }
963
964 #[tokio::test]
965 async fn test_copy_from_s3_paths_without_relisting() {
966 use std::sync::Mutex;
967
968 use axum::Router;
969 use axum::http::{Method, StatusCode, Uri};
970 use axum::response::IntoResponse;
971
972 let requests = Arc::new(Mutex::new(Vec::new()));
973 let recorded = requests.clone();
974 let app = Router::new().fallback(move |method: Method, uri: Uri| {
975 let recorded = recorded.clone();
976 async move {
977 recorded.lock().unwrap().push((method.clone(), uri.clone()));
978 if method == Method::HEAD {
979 let status = if uri.path().ends_with("missing.parquet") {
980 StatusCode::NOT_FOUND
981 } else if uri.path().ends_with("denied.parquet") {
982 StatusCode::FORBIDDEN
983 } else {
984 StatusCode::OK
985 };
986 return (status, [("content-length", "0")]).into_response();
987 }
988 assert_eq!(method, Method::GET);
989 let query =
990 axum::extract::Query::<HashMap<String, String>>::try_from_uri(&uri).unwrap();
991 assert_eq!(query.get("prefix").unwrap(), "backup/schema/1/");
992 assert_eq!(query.get("list-type").unwrap(), "2");
993 let files = (0..16).map(|i| format!(
994 "<Contents><Key>backup/schema/1/data.{i}.parquet</Key><LastModified>2026-09-01T00:00:00Z</LastModified><Size>0</Size></Contents>"
995 )).collect::<String>();
996 format!(
997 "<ListBucketResult><IsTruncated>false</IsTruncated>{files}\
998 <Contents><Key>backup/schema/1/other.csv</Key><LastModified>2026-09-01T00:00:00Z</LastModified><Size>0</Size></Contents>\
999 <CommonPrefixes><Prefix>backup/schema/1/data.dir/</Prefix></CommonPrefixes>\
1000 </ListBucketResult>"
1001 )
1002 .into_response()
1003 }
1004 });
1005 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1006 let endpoint = format!("http://{}", listener.local_addr().unwrap());
1007 let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
1008 let connection = HashMap::from([
1009 ("endpoint".into(), endpoint),
1010 ("region".into(), "us-east-1".into()),
1011 ("access_key_id".into(), "test".into()),
1012 ("secret_access_key".into(), "test".into()),
1013 ("disable_ec2_metadata".into(), "true".into()),
1014 ]);
1015 for i in 0..16 {
1016 let filename = format!("data.{i}.parquet");
1017 let mut req = copy_from_request(
1018 &format!("s3://bucket/backup/schema/1/{filename}"),
1019 Some("does-not-match"),
1020 );
1021 req.connection = connection.clone();
1022 let (_, paths) = list_copy_from_paths(&req, &LocalFileAccess::Disabled)
1023 .await
1024 .unwrap();
1025 assert_eq!(paths, [filename]);
1026 }
1027 {
1028 let requests = requests.lock().unwrap();
1029 assert_eq!(requests.len(), 16);
1030 assert!(requests.iter().all(|(method, uri)| method == Method::HEAD
1031 && uri.path().starts_with("/bucket/backup/schema/1/")));
1032 }
1033 let mut req = copy_from_request("s3://bucket/backup/schema/1/", Some(r"^data\."));
1034 req.connection = connection.clone();
1035 let (_, mut paths) = list_copy_from_paths(&req, &LocalFileAccess::Disabled)
1036 .await
1037 .unwrap();
1038 paths.sort();
1039 let mut expected = (0..16)
1040 .map(|i| format!("data.{i}.parquet"))
1041 .collect::<Vec<_>>();
1042 expected.sort();
1043 assert_eq!(paths, expected);
1044 assert_eq!(
1045 requests
1046 .lock()
1047 .unwrap()
1048 .iter()
1049 .filter(|(method, _)| *method == Method::GET)
1050 .count(),
1051 1
1052 );
1053
1054 for (name, kind) in [
1055 ("missing", object_store::ErrorKind::NotFound),
1056 ("denied", object_store::ErrorKind::PermissionDenied),
1057 ] {
1058 let mut req =
1059 copy_from_request(&format!("s3://bucket/backup/schema/1/{name}.parquet"), None);
1060 req.connection = connection.clone();
1061 assert!(matches!(
1062 list_copy_from_paths(&req, &LocalFileAccess::Disabled).await,
1063 Err(error::Error::ListObjects {
1064 source: common_datasource::error::Error::ListObjects { error, .. }, ..
1065 }) if error.kind() == kind
1066 ));
1067 }
1068 server.abort();
1069 }
1070
1071 fn test_schema_matches(from: (DataType, bool), to: (DataType, bool), matches: bool) {
1072 let s1 = Arc::new(Schema::new(vec![Field::new("col", from.0.clone(), from.1)]));
1073 let s2 = Arc::new(Schema::new(vec![Field::new("col", to.0.clone(), to.1)]));
1074 let res = ensure_schema_compatible(&s1, &s2);
1075 assert_eq!(
1076 matches,
1077 res.is_ok(),
1078 "from data type: {}, to data type: {}, expected: {}, but got: {}",
1079 from.0,
1080 to.0,
1081 matches,
1082 res.is_ok()
1083 )
1084 }
1085
1086 #[test]
1087 fn test_ensure_datatype_matches_ignore_timezone() {
1088 test_schema_matches(
1089 (
1090 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
1091 true,
1092 ),
1093 (
1094 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
1095 true,
1096 ),
1097 true,
1098 );
1099
1100 test_schema_matches(
1101 (
1102 DataType::Timestamp(
1103 datatypes::arrow::datatypes::TimeUnit::Second,
1104 Some("UTC".into()),
1105 ),
1106 true,
1107 ),
1108 (
1109 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
1110 true,
1111 ),
1112 true,
1113 );
1114
1115 test_schema_matches(
1116 (
1117 DataType::Timestamp(
1118 datatypes::arrow::datatypes::TimeUnit::Second,
1119 Some("UTC".into()),
1120 ),
1121 true,
1122 ),
1123 (
1124 DataType::Timestamp(
1125 datatypes::arrow::datatypes::TimeUnit::Second,
1126 Some("PDT".into()),
1127 ),
1128 true,
1129 ),
1130 true,
1131 );
1132
1133 test_schema_matches(
1134 (
1135 DataType::Timestamp(
1136 datatypes::arrow::datatypes::TimeUnit::Second,
1137 Some("UTC".into()),
1138 ),
1139 true,
1140 ),
1141 (
1142 DataType::Timestamp(
1143 datatypes::arrow::datatypes::TimeUnit::Millisecond,
1144 Some("UTC".into()),
1145 ),
1146 true,
1147 ),
1148 true,
1149 );
1150
1151 test_schema_matches((DataType::Int8, true), (DataType::Int8, true), true);
1152
1153 test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
1154 }
1155
1156 #[test]
1157 fn test_data_type_equals_ignore_timezone_with_options() {
1158 test_schema_matches(
1159 (
1160 DataType::Timestamp(
1161 datatypes::arrow::datatypes::TimeUnit::Microsecond,
1162 Some("UTC".into()),
1163 ),
1164 true,
1165 ),
1166 (
1167 DataType::Timestamp(
1168 datatypes::arrow::datatypes::TimeUnit::Millisecond,
1169 Some("PDT".into()),
1170 ),
1171 true,
1172 ),
1173 true,
1174 );
1175
1176 test_schema_matches(
1177 (DataType::Utf8, true),
1178 (
1179 DataType::Timestamp(
1180 datatypes::arrow::datatypes::TimeUnit::Millisecond,
1181 Some("PDT".into()),
1182 ),
1183 true,
1184 ),
1185 true,
1186 );
1187
1188 test_schema_matches(
1189 (
1190 DataType::Timestamp(
1191 datatypes::arrow::datatypes::TimeUnit::Millisecond,
1192 Some("PDT".into()),
1193 ),
1194 true,
1195 ),
1196 (DataType::Utf8, true),
1197 true,
1198 );
1199 }
1200
1201 #[test]
1202 fn test_map_to_binary_json_compatibility() {
1203 let map_type = DataType::Map(
1205 Arc::new(Field::new(
1206 "key_value",
1207 DataType::Struct(
1208 vec![
1209 Field::new("key", DataType::Utf8, false),
1210 Field::new("value", DataType::Utf8, false),
1211 ]
1212 .into(),
1213 ),
1214 false,
1215 )),
1216 false,
1217 );
1218
1219 test_schema_matches((map_type, false), (DataType::Binary, true), true);
1220
1221 test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
1222 test_schema_matches((DataType::Utf8, true), (DataType::Binary, true), true);
1223 }
1224
1225 fn make_test_schema(v: &[Field]) -> Arc<Schema> {
1226 Arc::new(Schema::new(v.to_vec()))
1227 }
1228
1229 #[test]
1230 fn test_compatible_file_schema() {
1231 let file_schema0 = make_test_schema(&[
1232 Field::new("c1", DataType::UInt8, true),
1233 Field::new("c2", DataType::UInt8, true),
1234 ]);
1235
1236 let table_schema = make_test_schema(&[
1237 Field::new("c1", DataType::Int16, true),
1238 Field::new("c2", DataType::Int16, true),
1239 Field::new("c3", DataType::Int16, true),
1240 ]);
1241
1242 let compat_schema = make_test_schema(&[
1243 Field::new("c1", DataType::Int16, true),
1244 Field::new("c2", DataType::Int16, true),
1245 ]);
1246
1247 let (_, tp, _) =
1248 generated_schema_projection_and_compatible_file_schema(&file_schema0, &table_schema);
1249
1250 assert_eq!(table_schema.project(&tp).unwrap(), *compat_schema);
1251 }
1252
1253 #[test]
1254 fn test_schema_projection() {
1255 let file_schema0 = make_test_schema(&[
1256 Field::new("c1", DataType::UInt8, true),
1257 Field::new("c2", DataType::UInt8, true),
1258 Field::new("c3", DataType::UInt8, true),
1259 ]);
1260
1261 let file_schema1 = make_test_schema(&[
1262 Field::new("c3", DataType::UInt8, true),
1263 Field::new("c4", DataType::UInt8, true),
1264 ]);
1265
1266 let file_schema2 = make_test_schema(&[
1267 Field::new("c3", DataType::UInt8, true),
1268 Field::new("c4", DataType::UInt8, true),
1269 Field::new("c5", DataType::UInt8, true),
1270 ]);
1271
1272 let file_schema3 = make_test_schema(&[
1273 Field::new("c1", DataType::UInt8, true),
1274 Field::new("c2", DataType::UInt8, true),
1275 ]);
1276
1277 let table_schema = make_test_schema(&[
1278 Field::new("c3", DataType::UInt8, true),
1279 Field::new("c4", DataType::UInt8, true),
1280 Field::new("c5", DataType::UInt8, true),
1281 ]);
1282
1283 let tests = [
1284 (&file_schema0, &table_schema, true), (&file_schema1, &table_schema, true), (&file_schema2, &table_schema, true), (&file_schema3, &table_schema, true), ];
1289
1290 for test in tests {
1291 let (fp, tp, _) =
1292 generated_schema_projection_and_compatible_file_schema(test.0, test.1);
1293 assert_eq!(test.0.project(&fp).unwrap(), test.1.project(&tp).unwrap());
1294 }
1295 }
1296
1297 #[test]
1298 fn test_csv_reader_schema_for_skip_bad_records() {
1299 let file_schema = make_test_schema(&[
1300 Field::new("id", DataType::Utf8, true),
1301 Field::new("jsons", DataType::Utf8, true),
1302 Field::new("ts", DataType::Utf8, true),
1303 ]);
1304 let compat_schema = make_test_schema(&[
1305 Field::new("id", DataType::UInt32, true),
1306 Field::new("jsons", DataType::Binary, true),
1307 Field::new(
1308 "ts",
1309 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1310 true,
1311 ),
1312 ]);
1313
1314 let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1315
1316 assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1317 assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1318 assert_eq!(
1319 reader_schema.field(2).data_type(),
1320 compat_schema.field(2).data_type()
1321 );
1322 }
1323
1324 fn make_csv_metadata(schema: Arc<Schema>, has_header: bool) -> FileMetadata {
1325 FileMetadata::Csv {
1326 schema,
1327 format: CsvFormat {
1328 has_header,
1329 ..CsvFormat::default()
1330 },
1331 path: "test.csv".to_string(),
1332 }
1333 }
1334
1335 fn make_strict_csv_metadata(schema: Arc<Schema>) -> FileMetadata {
1336 FileMetadata::Csv {
1337 schema,
1338 format: CsvFormat {
1339 strict_headers: true,
1340 ..CsvFormat::default()
1341 },
1342 path: "test.csv".to_string(),
1343 }
1344 }
1345
1346 fn assert_field(schema: &Schema, idx: usize, name: &str, data_type: &DataType) {
1347 let field = schema.field(idx);
1348 assert_eq!(field.name(), name);
1349 assert_eq!(field.data_type(), data_type);
1350 }
1351
1352 #[test]
1353 fn test_strict_csv_headers_allows_reordered_columns() {
1354 let file_schema = make_test_schema(&[
1355 Field::new("ts", DataType::Utf8, true),
1356 Field::new("host_id", DataType::UInt8, true),
1357 Field::new("reading_value", DataType::Float64, true),
1358 ]);
1359 let table_schema = make_test_schema(&[
1360 Field::new("host_id", DataType::UInt32, true),
1361 Field::new("reading_value", DataType::Float64, true),
1362 Field::new("ts", DataType::Utf8, true),
1363 ]);
1364
1365 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1366 .unwrap();
1367 }
1368
1369 #[test]
1370 fn test_strict_csv_headers_rejects_unknown_columns() {
1371 let file_schema = make_test_schema(&[
1372 Field::new("host_id", DataType::UInt8, true),
1373 Field::new("reading_value", DataType::Float64, true),
1374 Field::new("ts", DataType::Utf8, true),
1375 Field::new("extra", DataType::Utf8, true),
1376 ]);
1377 let table_schema = make_test_schema(&[
1378 Field::new("host_id", DataType::UInt32, true),
1379 Field::new("reading_value", DataType::Float64, true),
1380 Field::new("ts", DataType::Utf8, true),
1381 ]);
1382
1383 let err =
1384 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1385 .unwrap_err();
1386
1387 assert!(matches!(
1388 err,
1389 error::Error::CsvHeaderMismatch {
1390 unknown_columns,
1391 missing_columns,
1392 duplicate_columns,
1393 ..
1394 } if unknown_columns == vec!["extra".to_string()]
1395 && missing_columns.is_empty()
1396 && duplicate_columns.is_empty()
1397 ));
1398 }
1399
1400 #[test]
1401 fn test_strict_csv_headers_rejects_missing_columns() {
1402 let file_schema = make_test_schema(&[
1403 Field::new("host_id", DataType::UInt8, true),
1404 Field::new("ts", DataType::Utf8, true),
1405 ]);
1406 let table_schema = make_test_schema(&[
1407 Field::new("host_id", DataType::UInt32, true),
1408 Field::new("reading_value", DataType::Float64, true),
1409 Field::new("ts", DataType::Utf8, true),
1410 ]);
1411
1412 let err =
1413 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1414 .unwrap_err();
1415
1416 assert!(matches!(
1417 err,
1418 error::Error::CsvHeaderMismatch {
1419 unknown_columns,
1420 missing_columns,
1421 duplicate_columns,
1422 ..
1423 } if unknown_columns.is_empty()
1424 && missing_columns == vec!["reading_value".to_string()]
1425 && duplicate_columns.is_empty()
1426 ));
1427 }
1428
1429 #[test]
1430 fn test_strict_csv_headers_rejects_duplicate_columns() {
1431 let file_schema = make_test_schema(&[
1432 Field::new("host_id", DataType::UInt8, true),
1433 Field::new("reading_value", DataType::Float64, true),
1434 Field::new("ts", DataType::Utf8, true),
1435 Field::new("host_id", DataType::UInt16, true),
1436 ]);
1437 let table_schema = make_test_schema(&[
1438 Field::new("host_id", DataType::UInt32, true),
1439 Field::new("reading_value", DataType::Float64, true),
1440 Field::new("ts", DataType::Utf8, true),
1441 ]);
1442
1443 let err =
1444 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1445 .unwrap_err();
1446
1447 assert!(matches!(
1448 err,
1449 error::Error::CsvHeaderMismatch {
1450 unknown_columns,
1451 missing_columns,
1452 duplicate_columns,
1453 ..
1454 } if unknown_columns.is_empty()
1455 && missing_columns.is_empty()
1456 && duplicate_columns == vec!["host_id".to_string()]
1457 ));
1458 }
1459
1460 #[test]
1461 fn test_headerless_csv_schema_projection_is_positional() {
1462 let file_schema = make_test_schema(&[
1463 Field::new("column_1", DataType::UInt8, true),
1464 Field::new("column_2", DataType::Float64, true),
1465 Field::new("column_3", DataType::Utf8, true),
1466 ]);
1467 let table_schema = make_test_schema(&[
1468 Field::new("host_id", DataType::UInt32, true),
1469 Field::new("reading_value", DataType::Float64, true),
1470 Field::new(
1471 "ts",
1472 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1473 true,
1474 ),
1475 ]);
1476
1477 let mapping =
1478 copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1479
1480 assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1481 assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1482 assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1483 assert_field(
1484 &mapping.compat_file_schema,
1485 1,
1486 "reading_value",
1487 &DataType::Float64,
1488 );
1489 assert_field(
1490 &mapping.compat_file_schema,
1491 2,
1492 "ts",
1493 table_schema.field(2).data_type(),
1494 );
1495 assert_eq!(
1496 mapping
1497 .compat_file_schema
1498 .project(&mapping.file_projection)
1499 .unwrap(),
1500 table_schema.project(&mapping.table_projection).unwrap()
1501 );
1502 }
1503
1504 #[test]
1505 fn test_headerless_csv_schema_projection_ignores_extra_file_columns() {
1506 let file_schema = make_test_schema(&[
1507 Field::new("column_1", DataType::UInt8, true),
1508 Field::new("column_2", DataType::Float64, true),
1509 Field::new("column_3", DataType::Utf8, true),
1510 Field::new("column_4", DataType::Utf8, true),
1511 ]);
1512 let table_schema = make_test_schema(&[
1513 Field::new("host_id", DataType::UInt32, true),
1514 Field::new("reading_value", DataType::Float64, true),
1515 Field::new("ts", DataType::Utf8, true),
1516 ]);
1517
1518 let mapping =
1519 copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1520
1521 assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1522 assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1523 assert_eq!(mapping.compat_file_schema.fields().len(), 4);
1524 assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1525 assert_field(
1526 &mapping.compat_file_schema,
1527 1,
1528 "reading_value",
1529 &DataType::Float64,
1530 );
1531 assert_field(&mapping.compat_file_schema, 2, "ts", &DataType::Utf8);
1532 assert_field(&mapping.compat_file_schema, 3, "column_4", &DataType::Utf8);
1533 }
1534
1535 #[test]
1536 fn test_headerless_csv_schema_projection_supports_prefix_import() {
1537 let file_schema = make_test_schema(&[
1538 Field::new("column_1", DataType::UInt8, true),
1539 Field::new("column_2", DataType::Float64, true),
1540 ]);
1541 let table_schema = make_test_schema(&[
1542 Field::new("host_id", DataType::UInt32, true),
1543 Field::new("reading_value", DataType::Float64, true),
1544 Field::new("ts", DataType::Utf8, true),
1545 ]);
1546
1547 let mapping =
1548 copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1549
1550 assert_eq!(mapping.file_projection, vec![0, 1]);
1551 assert_eq!(mapping.table_projection, vec![0, 1]);
1552 assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1553 assert_field(
1554 &mapping.compat_file_schema,
1555 1,
1556 "reading_value",
1557 &DataType::Float64,
1558 );
1559 assert_eq!(
1560 mapping
1561 .compat_file_schema
1562 .project(&mapping.file_projection)
1563 .unwrap(),
1564 table_schema.project(&mapping.table_projection).unwrap()
1565 );
1566 }
1567
1568 #[test]
1569 fn test_csv_reader_schema_for_skip_bad_records_uses_positional_mapping() {
1570 let file_schema = make_test_schema(&[
1571 Field::new("column_1", DataType::Utf8, true),
1572 Field::new("column_2", DataType::Utf8, true),
1573 Field::new("column_3", DataType::Utf8, true),
1574 ]);
1575 let table_schema = make_test_schema(&[
1576 Field::new("host_id", DataType::UInt32, true),
1577 Field::new("jsons", DataType::Binary, true),
1578 Field::new(
1579 "ts",
1580 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1581 true,
1582 ),
1583 ]);
1584 let mapping = copy_from_schema_mapping(
1585 &make_csv_metadata(file_schema.clone(), false),
1586 &table_schema,
1587 );
1588 let compat_schema = Arc::new(mapping.compat_file_schema);
1589
1590 let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1591
1592 assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1593 assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1594 assert_eq!(
1595 reader_schema.field(2).data_type(),
1596 table_schema.field(2).data_type()
1597 );
1598 }
1599}