1use std::collections::{BTreeSet, HashMap, HashSet};
16use std::future::Future;
17use std::pin::Pin;
18use std::sync::Arc;
19use std::task::{Context, Poll};
20
21use client::{Output, OutputData, OutputMeta};
22use common_base::readable_size::ReadableSize;
23use common_datasource::file_format::csv::{
24 CsvFormat, is_skippable_arrow_error, tolerant_csv_stream,
25};
26use common_datasource::file_format::json::JsonFormat;
27use common_datasource::file_format::orc::{ReaderAdapter, infer_orc_schema, new_orc_stream_reader};
28use common_datasource::file_format::{FileFormat, Format, file_to_stream};
29use common_datasource::lister::{Lister, Source};
30use common_datasource::object_store::build_backend_with_path;
31use common_query::{OutputCost, OutputRows};
32use common_recordbatch::DfSendableRecordBatchStream;
33use common_recordbatch::adapter::RecordBatchStreamTypeAdapter;
34use common_telemetry::{debug, tracing};
35use datafusion::datasource::physical_plan::{CsvSource, FileSource, JsonSource};
36use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
37use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata;
38use datafusion_common::DataFusionError;
39use datafusion_common::arrow::error::ArrowError;
40use datafusion_common::config::CsvOptions;
41use datafusion_expr::Expr;
42use datatypes::arrow::compute::can_cast_types;
43use datatypes::arrow::datatypes::{DataType as ArrowDataType, Schema, SchemaRef};
44use datatypes::arrow::record_batch::RecordBatch;
45use datatypes::vectors::Helper;
46use futures_util::StreamExt;
47use object_store::{Entry, EntryMode, ObjectStore};
48use regex::Regex;
49use session::context::QueryContextRef;
50use snafu::{ResultExt, ensure};
51use table::requests::{CopyTableRequest, InsertRequest};
52use table::table_reference::TableReference;
53use tokio_util::compat::FuturesAsyncReadCompatExt;
54
55use crate::error::{self, IntoVectorsSnafu, Result};
56use crate::statement::StatementExecutor;
57
58const DEFAULT_BATCH_SIZE: usize = 8192;
59const DEFAULT_READ_BUFFER: usize = 256 * 1024;
60
61enum FileMetadata {
62 Parquet {
63 schema: SchemaRef,
64 metadata: ArrowReaderMetadata,
65 path: String,
66 },
67 Orc {
68 schema: SchemaRef,
69 path: String,
70 },
71 Json {
72 schema: SchemaRef,
73 format: JsonFormat,
74 path: String,
75 },
76 Csv {
77 schema: SchemaRef,
78 format: CsvFormat,
79 path: String,
80 },
81}
82
83impl FileMetadata {
84 pub fn schema(&self) -> &SchemaRef {
86 match self {
87 FileMetadata::Parquet { schema, .. } => schema,
88 FileMetadata::Orc { schema, .. } => schema,
89 FileMetadata::Json { schema, .. } => schema,
90 FileMetadata::Csv { schema, .. } => schema,
91 }
92 }
93}
94
95impl StatementExecutor {
96 async fn list_copy_from_entries(
97 &self,
98 req: &CopyTableRequest,
99 ) -> Result<(ObjectStore, Vec<Entry>)> {
100 let backend =
101 build_backend_with_path(&req.location, &req.connection, &self.local_file_access)
102 .await
103 .context(error::BuildBackendSnafu)?;
104 let regex = req
105 .pattern
106 .as_ref()
107 .map(|x| Regex::new(x))
108 .transpose()
109 .context(error::BuildRegexSnafu)?;
110
111 let source = if let Some(filename) = backend.object_path {
112 Source::Filename(filename)
113 } else {
114 Source::Dir
115 };
116
117 let lister = Lister::new(
118 backend.object_store.clone(),
119 source.clone(),
120 req.location.clone(),
121 regex,
122 );
123
124 let entries = lister.list().await.context(error::ListObjectsSnafu)?;
125 debug!(
126 "Copy from location: {:?}, {source:?}, entries: {entries:?}",
127 req.location
128 );
129 Ok((backend.object_store, entries))
130 }
131
132 async fn collect_metadata(
133 &self,
134 object_store: &ObjectStore,
135 format: Format,
136 path: String,
137 ) -> Result<FileMetadata> {
138 match format {
139 Format::Csv(format) => Ok(FileMetadata::Csv {
140 schema: Arc::new(
141 format
142 .infer_schema(object_store, &path)
143 .await
144 .context(error::InferSchemaSnafu { path: &path })?,
145 ),
146 format,
147 path,
148 }),
149 Format::Json(format) => Ok(FileMetadata::Json {
150 schema: Arc::new(
151 format
152 .infer_schema(object_store, &path)
153 .await
154 .context(error::InferSchemaSnafu { path: &path })?,
155 ),
156 format,
157 path,
158 }),
159 Format::Parquet(_) => {
160 let meta = object_store
161 .stat(&path)
162 .await
163 .context(error::ReadObjectSnafu { path: &path })?;
164 let mut reader = object_store
165 .reader(&path)
166 .await
167 .context(error::ReadObjectSnafu { path: &path })?
168 .into_futures_async_read(0..meta.content_length())
169 .await
170 .context(error::ReadObjectSnafu { path: &path })?
171 .compat();
172 let metadata = ArrowReaderMetadata::load_async(&mut reader, Default::default())
173 .await
174 .context(error::ReadParquetMetadataSnafu)?;
175
176 Ok(FileMetadata::Parquet {
177 schema: metadata.schema().clone(),
178 metadata,
179 path,
180 })
181 }
182 Format::Orc(_) => {
183 let meta = object_store
184 .stat(&path)
185 .await
186 .context(error::ReadObjectSnafu { path: &path })?;
187
188 let reader = object_store
189 .reader(&path)
190 .await
191 .context(error::ReadObjectSnafu { path: &path })?;
192
193 let schema = infer_orc_schema(ReaderAdapter::new(reader, meta.content_length()))
194 .await
195 .context(error::ReadOrcSnafu)?;
196
197 Ok(FileMetadata::Orc {
198 schema: Arc::new(schema),
199 path,
200 })
201 }
202 }
203 }
204
205 async fn build_read_stream(
206 &self,
207 compat_schema: SchemaRef,
208 object_store: &ObjectStore,
209 file_metadata: &FileMetadata,
210 projection: Vec<usize>,
211 filters: Vec<Expr>,
212 ) -> Result<DfSendableRecordBatchStream> {
213 match file_metadata {
214 FileMetadata::Csv {
215 format,
216 path,
217 schema,
218 } => {
219 let output_schema = Arc::new(
220 compat_schema
221 .project(&projection)
222 .context(error::ProjectSchemaSnafu)?,
223 );
224
225 let options = CsvOptions::default()
226 .with_has_header(format.has_header)
227 .with_delimiter(format.delimiter);
228 let csv_source = CsvSource::new(schema.clone())
229 .with_csv_options(options)
230 .with_batch_size(DEFAULT_BATCH_SIZE);
231 let stream = if format.skip_bad_records {
232 let reader_schema =
233 csv_reader_schema_for_skip_bad_records(schema, &compat_schema);
234 tolerant_csv_stream(
235 object_store,
236 path,
237 Arc::new(reader_schema),
238 projection.clone(),
239 format,
240 )
241 .await
242 .context(error::BuildFileStreamSnafu)?
243 } else {
244 file_to_stream(
245 object_store,
246 path,
247 csv_source,
248 Some(projection),
249 format.compression_type,
250 )
251 .await
252 .context(error::BuildFileStreamSnafu)?
253 };
254
255 let stream = Box::pin(
256 RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
259 .with_filter(filters)
260 .context(error::PhysicalExprSnafu)?,
261 );
262 if format.skip_bad_records {
263 Ok(Box::pin(SkipBadRecordsStream::new(stream, path)))
264 } else {
265 Ok(stream)
266 }
267 }
268 FileMetadata::Json {
269 path,
270 format,
271 schema,
272 } => {
273 let output_schema = Arc::new(
274 compat_schema
275 .project(&projection)
276 .context(error::ProjectSchemaSnafu)?,
277 );
278
279 let json_source =
280 JsonSource::new(schema.clone()).with_batch_size(DEFAULT_BATCH_SIZE);
281 let stream = file_to_stream(
282 object_store,
283 path,
284 json_source,
285 Some(projection),
286 format.compression_type,
287 )
288 .await
289 .context(error::BuildFileStreamSnafu)?;
290
291 Ok(Box::pin(
292 RecordBatchStreamTypeAdapter::new(output_schema, stream, None)
295 .with_filter(filters)
296 .context(error::PhysicalExprSnafu)?,
297 ))
298 }
299 FileMetadata::Parquet { metadata, path, .. } => {
300 let meta = object_store
301 .stat(path)
302 .await
303 .context(error::ReadObjectSnafu { path })?;
304 let reader = object_store
305 .reader_with(path)
306 .chunk(DEFAULT_READ_BUFFER)
307 .await
308 .context(error::ReadObjectSnafu { path })?
309 .into_futures_async_read(0..meta.content_length())
310 .await
311 .context(error::ReadObjectSnafu { path })?
312 .compat();
313 let builder =
314 ParquetRecordBatchStreamBuilder::new_with_metadata(reader, metadata.clone());
315 let stream = builder
316 .build()
317 .context(error::BuildParquetRecordBatchStreamSnafu)?;
318
319 let output_schema = Arc::new(
320 compat_schema
321 .project(&projection)
322 .context(error::ProjectSchemaSnafu)?,
323 );
324 Ok(Box::pin(
325 RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
326 .with_filter(filters)
327 .context(error::PhysicalExprSnafu)?,
328 ))
329 }
330 FileMetadata::Orc { path, .. } => {
331 let meta = object_store
332 .stat(path)
333 .await
334 .context(error::ReadObjectSnafu { path })?;
335
336 let reader = object_store
337 .reader_with(path)
338 .chunk(DEFAULT_READ_BUFFER)
339 .await
340 .context(error::ReadObjectSnafu { path })?;
341 let stream =
342 new_orc_stream_reader(ReaderAdapter::new(reader, meta.content_length()))
343 .await
344 .context(error::ReadOrcSnafu)?;
345
346 let output_schema = Arc::new(
347 compat_schema
348 .project(&projection)
349 .context(error::ProjectSchemaSnafu)?,
350 );
351
352 Ok(Box::pin(
353 RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection))
354 .with_filter(filters)
355 .context(error::PhysicalExprSnafu)?,
356 ))
357 }
358 }
359 }
360
361 #[tracing::instrument(skip_all)]
362 pub async fn copy_table_from(
363 &self,
364 req: CopyTableRequest,
365 query_ctx: QueryContextRef,
366 ) -> Result<Output> {
367 let table_ref = TableReference {
368 catalog: &req.catalog_name,
369 schema: &req.schema_name,
370 table: &req.table_name,
371 };
372 let table = self.get_table(&table_ref).await?;
373 let format = Format::try_from(&req.with).context(error::ParseFileFormatSnafu)?;
374 let (object_store, entries) = self.list_copy_from_entries(&req).await?;
375 let mut files = Vec::with_capacity(entries.len());
376 let table_schema = table.schema().arrow_schema().clone();
377 let filters = table
378 .schema()
379 .timestamp_column()
380 .and_then(|c| {
381 common_query::logical_plan::build_same_type_ts_filter(c, req.timestamp_range)
382 })
383 .into_iter()
384 .collect::<Vec<_>>();
385
386 for entry in entries.iter() {
387 if entry.metadata().mode() != EntryMode::FILE {
388 continue;
389 }
390 let path = entry.path();
391 let file_metadata = self
392 .collect_metadata(&object_store, format.clone(), path.to_string())
393 .await?;
394
395 validate_csv_headers_if_required(&file_metadata, &table_schema)?;
396 let schema_mapping = copy_from_schema_mapping(&file_metadata, &table_schema);
397 let projected_file_schema = Arc::new(
398 file_metadata
399 .schema()
400 .project(&schema_mapping.file_projection)
401 .context(error::ProjectSchemaSnafu)?,
402 );
403 let projected_table_schema = Arc::new(
404 table_schema
405 .project(&schema_mapping.table_projection)
406 .context(error::ProjectSchemaSnafu)?,
407 );
408 ensure_schema_compatible(&projected_file_schema, &projected_table_schema)?;
409
410 files.push((
411 Arc::new(schema_mapping.compat_file_schema),
412 schema_mapping.file_projection,
413 projected_table_schema,
414 file_metadata,
415 ))
416 }
417
418 let mut rows_inserted = 0;
419 let mut insert_cost = 0;
420 let max_insert_rows = req
421 .limit
422 .map(|n| {
423 usize::try_from(n).map_err(|_| {
424 error::InvalidCopyParameterSnafu {
425 key: "limit".to_string(),
426 value: n.to_string(),
427 }
428 .build()
429 })
430 })
431 .transpose()?;
432 if max_insert_rows == Some(0) {
433 return Ok(gen_insert_output(rows_inserted, insert_cost));
434 }
435
436 let mut accepted_rows = 0;
437 for (compat_schema, file_schema_projection, projected_table_schema, file_metadata) in files
438 {
439 let mut stream = self
440 .build_read_stream(
441 compat_schema,
442 &object_store,
443 &file_metadata,
444 file_schema_projection,
445 filters.clone(),
446 )
447 .await?;
448
449 let fields = projected_table_schema
450 .fields()
451 .iter()
452 .map(|f| f.name().clone())
453 .collect::<Vec<_>>();
454
455 let pending_mem_threshold = ReadableSize::mb(32).as_bytes();
457 let mut pending_mem_size = 0;
458 let mut pending = vec![];
459
460 while let Some(r) = stream.next().await {
461 let record_batch = r.context(error::ReadDfRecordBatchSnafu)?;
462 let record_batch = if let Some(max_insert_rows) = max_insert_rows {
463 let remaining_rows = max_insert_rows - accepted_rows;
464 if record_batch.num_rows() > remaining_rows {
465 record_batch.slice(0, remaining_rows)
466 } else {
467 record_batch
468 }
469 } else {
470 record_batch
471 };
472 let record_batch_rows = record_batch.num_rows();
473 let vectors =
474 Helper::try_into_vectors(record_batch.columns()).context(IntoVectorsSnafu)?;
475
476 pending_mem_size += vectors.iter().map(|v| v.memory_size()).sum::<usize>();
477
478 let columns_values = fields
479 .iter()
480 .cloned()
481 .zip(vectors)
482 .collect::<HashMap<_, _>>();
483
484 pending.push(self.inserter.handle_table_insert(
485 InsertRequest {
486 catalog_name: req.catalog_name.clone(),
487 schema_name: req.schema_name.clone(),
488 table_name: req.table_name.clone(),
489 columns_values,
490 skip_wal: query_ctx.skip_wal(),
491 },
492 query_ctx.clone(),
493 ));
494 accepted_rows += record_batch_rows;
495
496 if pending_mem_size as u64 >= pending_mem_threshold {
497 let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
498 rows_inserted += rows;
499 insert_cost += cost;
500 }
501
502 if let Some(max_insert_rows) = max_insert_rows
503 && accepted_rows == max_insert_rows
504 {
505 if !pending.is_empty() {
506 let (rows, cost) =
507 batch_insert(&mut pending, &mut pending_mem_size).await?;
508 rows_inserted += rows;
509 insert_cost += cost;
510 }
511 return Ok(gen_insert_output(rows_inserted, insert_cost));
512 }
513 }
514
515 if !pending.is_empty() {
516 let (rows, cost) = batch_insert(&mut pending, &mut pending_mem_size).await?;
517 rows_inserted += rows;
518 insert_cost += cost;
519 }
520 }
521
522 Ok(gen_insert_output(rows_inserted, insert_cost))
523 }
524}
525
526fn gen_insert_output(rows_inserted: usize, insert_cost: usize) -> Output {
527 Output::new(
528 OutputData::AffectedRows(rows_inserted),
529 OutputMeta::new_with_cost(insert_cost),
530 )
531}
532
533struct SkipBadRecordsStream {
534 inner: DfSendableRecordBatchStream,
535 path: String,
536}
537
538impl SkipBadRecordsStream {
539 fn new(inner: DfSendableRecordBatchStream, path: impl Into<String>) -> Self {
540 Self {
541 inner,
542 path: path.into(),
543 }
544 }
545}
546
547impl datafusion::physical_plan::RecordBatchStream for SkipBadRecordsStream {
548 fn schema(&self) -> SchemaRef {
549 self.inner.schema()
550 }
551}
552
553impl futures::Stream for SkipBadRecordsStream {
554 type Item = datafusion_common::Result<RecordBatch>;
555
556 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
557 let this = self.get_mut();
558 loop {
559 match this.inner.as_mut().poll_next(cx) {
560 Poll::Ready(Some(Err(error))) if is_skippable_record_error(&error) => {
561 common_telemetry::warn!(
562 "Skipping bad record while copying from {}: {}",
563 this.path,
564 error
565 );
566 continue;
567 }
568 other => return other,
569 }
570 }
571 }
572}
573
574fn is_skippable_record_error(error: &DataFusionError) -> bool {
575 match error {
576 DataFusionError::ArrowError(error, _) => is_skippable_arrow_error(error),
577 DataFusionError::External(error) => error
578 .downcast_ref::<ArrowError>()
579 .is_some_and(is_skippable_arrow_error),
580 DataFusionError::Context(_, error) => is_skippable_record_error(error),
581 _ => false,
582 }
583}
584
585async fn batch_insert(
587 pending: &mut Vec<impl Future<Output = Result<Output>>>,
588 pending_bytes: &mut usize,
589) -> Result<(OutputRows, OutputCost)> {
590 let batch = pending.drain(..);
591 let result = futures::future::try_join_all(batch)
592 .await?
593 .iter()
594 .map(|o| o.extract_rows_and_cost())
595 .reduce(|(a, b), (c, d)| (a + c, b + d))
596 .unwrap_or((0, 0));
597 *pending_bytes = 0;
598 Ok(result)
599}
600
601fn can_cast_types_for_greptime(from: &ArrowDataType, to: &ArrowDataType) -> bool {
603 if let ArrowDataType::Map(_, _) = from
605 && let ArrowDataType::Binary = to
606 {
607 return true;
608 }
609
610 can_cast_types(from, to)
612}
613
614fn csv_reader_schema_for_skip_bad_records(file: &SchemaRef, compat: &SchemaRef) -> Schema {
615 let fields = file
616 .fields()
617 .iter()
618 .enumerate()
619 .map(|(idx, file_field)| match compat.fields().get(idx) {
620 Some(compat_field) if can_csv_reader_parse_type(compat_field.data_type()) => {
621 compat_field.clone()
622 }
623 _ => file_field.clone(),
624 })
625 .collect::<Vec<_>>();
626
627 Schema::new_with_metadata(fields, file.metadata().clone())
628}
629
630fn can_csv_reader_parse_type(data_type: &ArrowDataType) -> bool {
631 match data_type {
632 ArrowDataType::Boolean
633 | ArrowDataType::Decimal32(_, _)
634 | ArrowDataType::Decimal64(_, _)
635 | ArrowDataType::Decimal128(_, _)
636 | ArrowDataType::Decimal256(_, _)
637 | ArrowDataType::Int8
638 | ArrowDataType::Int16
639 | ArrowDataType::Int32
640 | ArrowDataType::Int64
641 | ArrowDataType::UInt8
642 | ArrowDataType::UInt16
643 | ArrowDataType::UInt32
644 | ArrowDataType::UInt64
645 | ArrowDataType::Float32
646 | ArrowDataType::Float64
647 | ArrowDataType::Date32
648 | ArrowDataType::Date64
649 | ArrowDataType::Time32(_)
650 | ArrowDataType::Time64(_)
651 | ArrowDataType::Timestamp(_, _)
652 | ArrowDataType::Null
653 | ArrowDataType::Utf8
654 | ArrowDataType::Utf8View => true,
655 ArrowDataType::Dictionary(_, value_type) => value_type.as_ref() == &ArrowDataType::Utf8,
656 _ => false,
657 }
658}
659
660fn ensure_schema_compatible(from: &SchemaRef, to: &SchemaRef) -> Result<()> {
661 let not_match = from
662 .fields
663 .iter()
664 .zip(to.fields.iter())
665 .map(|(l, r)| (l.data_type(), r.data_type()))
666 .enumerate()
667 .find(|(_, (l, r))| !can_cast_types_for_greptime(l, r));
668
669 if let Some((index, _)) = not_match {
670 error::InvalidSchemaSnafu {
671 index,
672 table_schema: to.to_string(),
673 file_schema: from.to_string(),
674 }
675 .fail()
676 } else {
677 Ok(())
678 }
679}
680
681fn validate_csv_headers_if_required(file_metadata: &FileMetadata, table: &SchemaRef) -> Result<()> {
682 let FileMetadata::Csv {
683 schema,
684 format,
685 path,
686 } = file_metadata
687 else {
688 return Ok(());
689 };
690
691 if !format.strict_headers {
692 return Ok(());
693 }
694
695 let mut seen_file_columns = HashSet::with_capacity(schema.fields().len());
696 let duplicate_columns = schema
697 .fields()
698 .iter()
699 .filter_map(|field| {
700 if seen_file_columns.insert(field.name().clone()) {
701 None
702 } else {
703 Some(field.name().clone())
704 }
705 })
706 .collect::<BTreeSet<_>>()
707 .into_iter()
708 .collect::<Vec<_>>();
709 let file_columns = seen_file_columns.into_iter().collect::<BTreeSet<_>>();
710 let table_columns = table
711 .fields()
712 .iter()
713 .map(|field| field.name().clone())
714 .collect::<BTreeSet<_>>();
715 let unknown_columns = file_columns
716 .difference(&table_columns)
717 .cloned()
718 .collect::<Vec<_>>();
719 let missing_columns = table_columns
720 .difference(&file_columns)
721 .cloned()
722 .collect::<Vec<_>>();
723
724 ensure!(
725 unknown_columns.is_empty() && missing_columns.is_empty() && duplicate_columns.is_empty(),
726 error::CsvHeaderMismatchSnafu {
727 path,
728 unknown_columns,
729 missing_columns,
730 duplicate_columns,
731 }
732 );
733
734 Ok(())
735}
736
737fn generated_schema_projection_and_compatible_file_schema(
742 file: &SchemaRef,
743 table: &SchemaRef,
744) -> (Vec<usize>, Vec<usize>, Schema) {
745 let mut file_projection = Vec::with_capacity(file.fields.len());
746 let mut table_projection = Vec::with_capacity(file.fields.len());
747 let mut compatible_fields = file.fields.iter().cloned().collect::<Vec<_>>();
748 for (file_idx, file_field) in file.fields.iter().enumerate() {
749 if let Some((table_idx, table_field)) = table.fields.find(file_field.name()) {
750 file_projection.push(file_idx);
751 table_projection.push(table_idx);
752
753 compatible_fields[file_idx] = table_field.clone();
755 }
756 }
757
758 (
759 file_projection,
760 table_projection,
761 Schema::new(compatible_fields),
762 )
763}
764
765struct CopyFromSchemaMapping {
766 file_projection: Vec<usize>,
767 table_projection: Vec<usize>,
768 compat_file_schema: Schema,
769}
770
771fn copy_from_schema_mapping(
772 file_metadata: &FileMetadata,
773 table: &SchemaRef,
774) -> CopyFromSchemaMapping {
775 match file_metadata {
776 FileMetadata::Csv { schema, format, .. } if !format.has_header => {
777 generated_positional_schema_projection_and_compatible_file_schema(schema, table)
778 }
779 _ => {
780 let (file_projection, table_projection, compat_file_schema) =
781 generated_schema_projection_and_compatible_file_schema(
782 file_metadata.schema(),
783 table,
784 );
785 CopyFromSchemaMapping {
786 file_projection,
787 table_projection,
788 compat_file_schema,
789 }
790 }
791 }
792}
793
794fn generated_positional_schema_projection_and_compatible_file_schema(
795 file: &SchemaRef,
796 table: &SchemaRef,
797) -> CopyFromSchemaMapping {
798 let len = file.fields.len().min(table.fields.len());
799 let file_projection = (0..len).collect::<Vec<_>>();
800 let table_projection = (0..len).collect::<Vec<_>>();
801 let compatible_fields = file
802 .fields
803 .iter()
804 .enumerate()
805 .map(|(idx, file_field)| {
806 if idx < len {
807 table.fields[idx].clone()
808 } else {
809 file_field.clone()
810 }
811 })
812 .collect::<Vec<_>>();
813
814 CopyFromSchemaMapping {
815 file_projection,
816 table_projection,
817 compat_file_schema: Schema::new(compatible_fields),
818 }
819}
820
821#[cfg(test)]
822mod tests {
823 use std::sync::Arc;
824
825 use datatypes::arrow::datatypes::{DataType, Field, Schema};
826
827 use super::*;
828
829 fn test_schema_matches(from: (DataType, bool), to: (DataType, bool), matches: bool) {
830 let s1 = Arc::new(Schema::new(vec![Field::new("col", from.0.clone(), from.1)]));
831 let s2 = Arc::new(Schema::new(vec![Field::new("col", to.0.clone(), to.1)]));
832 let res = ensure_schema_compatible(&s1, &s2);
833 assert_eq!(
834 matches,
835 res.is_ok(),
836 "from data type: {}, to data type: {}, expected: {}, but got: {}",
837 from.0,
838 to.0,
839 matches,
840 res.is_ok()
841 )
842 }
843
844 #[test]
845 fn test_ensure_datatype_matches_ignore_timezone() {
846 test_schema_matches(
847 (
848 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
849 true,
850 ),
851 (
852 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
853 true,
854 ),
855 true,
856 );
857
858 test_schema_matches(
859 (
860 DataType::Timestamp(
861 datatypes::arrow::datatypes::TimeUnit::Second,
862 Some("UTC".into()),
863 ),
864 true,
865 ),
866 (
867 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Second, None),
868 true,
869 ),
870 true,
871 );
872
873 test_schema_matches(
874 (
875 DataType::Timestamp(
876 datatypes::arrow::datatypes::TimeUnit::Second,
877 Some("UTC".into()),
878 ),
879 true,
880 ),
881 (
882 DataType::Timestamp(
883 datatypes::arrow::datatypes::TimeUnit::Second,
884 Some("PDT".into()),
885 ),
886 true,
887 ),
888 true,
889 );
890
891 test_schema_matches(
892 (
893 DataType::Timestamp(
894 datatypes::arrow::datatypes::TimeUnit::Second,
895 Some("UTC".into()),
896 ),
897 true,
898 ),
899 (
900 DataType::Timestamp(
901 datatypes::arrow::datatypes::TimeUnit::Millisecond,
902 Some("UTC".into()),
903 ),
904 true,
905 ),
906 true,
907 );
908
909 test_schema_matches((DataType::Int8, true), (DataType::Int8, true), true);
910
911 test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
912 }
913
914 #[test]
915 fn test_data_type_equals_ignore_timezone_with_options() {
916 test_schema_matches(
917 (
918 DataType::Timestamp(
919 datatypes::arrow::datatypes::TimeUnit::Microsecond,
920 Some("UTC".into()),
921 ),
922 true,
923 ),
924 (
925 DataType::Timestamp(
926 datatypes::arrow::datatypes::TimeUnit::Millisecond,
927 Some("PDT".into()),
928 ),
929 true,
930 ),
931 true,
932 );
933
934 test_schema_matches(
935 (DataType::Utf8, true),
936 (
937 DataType::Timestamp(
938 datatypes::arrow::datatypes::TimeUnit::Millisecond,
939 Some("PDT".into()),
940 ),
941 true,
942 ),
943 true,
944 );
945
946 test_schema_matches(
947 (
948 DataType::Timestamp(
949 datatypes::arrow::datatypes::TimeUnit::Millisecond,
950 Some("PDT".into()),
951 ),
952 true,
953 ),
954 (DataType::Utf8, true),
955 true,
956 );
957 }
958
959 #[test]
960 fn test_map_to_binary_json_compatibility() {
961 let map_type = DataType::Map(
963 Arc::new(Field::new(
964 "key_value",
965 DataType::Struct(
966 vec![
967 Field::new("key", DataType::Utf8, false),
968 Field::new("value", DataType::Utf8, false),
969 ]
970 .into(),
971 ),
972 false,
973 )),
974 false,
975 );
976
977 test_schema_matches((map_type, false), (DataType::Binary, true), true);
978
979 test_schema_matches((DataType::Int8, true), (DataType::Int16, true), true);
980 test_schema_matches((DataType::Utf8, true), (DataType::Binary, true), true);
981 }
982
983 fn make_test_schema(v: &[Field]) -> Arc<Schema> {
984 Arc::new(Schema::new(v.to_vec()))
985 }
986
987 #[test]
988 fn test_compatible_file_schema() {
989 let file_schema0 = make_test_schema(&[
990 Field::new("c1", DataType::UInt8, true),
991 Field::new("c2", DataType::UInt8, true),
992 ]);
993
994 let table_schema = make_test_schema(&[
995 Field::new("c1", DataType::Int16, true),
996 Field::new("c2", DataType::Int16, true),
997 Field::new("c3", DataType::Int16, true),
998 ]);
999
1000 let compat_schema = make_test_schema(&[
1001 Field::new("c1", DataType::Int16, true),
1002 Field::new("c2", DataType::Int16, true),
1003 ]);
1004
1005 let (_, tp, _) =
1006 generated_schema_projection_and_compatible_file_schema(&file_schema0, &table_schema);
1007
1008 assert_eq!(table_schema.project(&tp).unwrap(), *compat_schema);
1009 }
1010
1011 #[test]
1012 fn test_schema_projection() {
1013 let file_schema0 = make_test_schema(&[
1014 Field::new("c1", DataType::UInt8, true),
1015 Field::new("c2", DataType::UInt8, true),
1016 Field::new("c3", DataType::UInt8, true),
1017 ]);
1018
1019 let file_schema1 = make_test_schema(&[
1020 Field::new("c3", DataType::UInt8, true),
1021 Field::new("c4", DataType::UInt8, true),
1022 ]);
1023
1024 let file_schema2 = make_test_schema(&[
1025 Field::new("c3", DataType::UInt8, true),
1026 Field::new("c4", DataType::UInt8, true),
1027 Field::new("c5", DataType::UInt8, true),
1028 ]);
1029
1030 let file_schema3 = make_test_schema(&[
1031 Field::new("c1", DataType::UInt8, true),
1032 Field::new("c2", DataType::UInt8, true),
1033 ]);
1034
1035 let table_schema = make_test_schema(&[
1036 Field::new("c3", DataType::UInt8, true),
1037 Field::new("c4", DataType::UInt8, true),
1038 Field::new("c5", DataType::UInt8, true),
1039 ]);
1040
1041 let tests = [
1042 (&file_schema0, &table_schema, true), (&file_schema1, &table_schema, true), (&file_schema2, &table_schema, true), (&file_schema3, &table_schema, true), ];
1047
1048 for test in tests {
1049 let (fp, tp, _) =
1050 generated_schema_projection_and_compatible_file_schema(test.0, test.1);
1051 assert_eq!(test.0.project(&fp).unwrap(), test.1.project(&tp).unwrap());
1052 }
1053 }
1054
1055 #[test]
1056 fn test_csv_reader_schema_for_skip_bad_records() {
1057 let file_schema = make_test_schema(&[
1058 Field::new("id", DataType::Utf8, true),
1059 Field::new("jsons", DataType::Utf8, true),
1060 Field::new("ts", DataType::Utf8, true),
1061 ]);
1062 let compat_schema = make_test_schema(&[
1063 Field::new("id", DataType::UInt32, true),
1064 Field::new("jsons", DataType::Binary, true),
1065 Field::new(
1066 "ts",
1067 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1068 true,
1069 ),
1070 ]);
1071
1072 let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1073
1074 assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1075 assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1076 assert_eq!(
1077 reader_schema.field(2).data_type(),
1078 compat_schema.field(2).data_type()
1079 );
1080 }
1081
1082 fn make_csv_metadata(schema: Arc<Schema>, has_header: bool) -> FileMetadata {
1083 FileMetadata::Csv {
1084 schema,
1085 format: CsvFormat {
1086 has_header,
1087 ..CsvFormat::default()
1088 },
1089 path: "test.csv".to_string(),
1090 }
1091 }
1092
1093 fn make_strict_csv_metadata(schema: Arc<Schema>) -> FileMetadata {
1094 FileMetadata::Csv {
1095 schema,
1096 format: CsvFormat {
1097 strict_headers: true,
1098 ..CsvFormat::default()
1099 },
1100 path: "test.csv".to_string(),
1101 }
1102 }
1103
1104 fn assert_field(schema: &Schema, idx: usize, name: &str, data_type: &DataType) {
1105 let field = schema.field(idx);
1106 assert_eq!(field.name(), name);
1107 assert_eq!(field.data_type(), data_type);
1108 }
1109
1110 #[test]
1111 fn test_strict_csv_headers_allows_reordered_columns() {
1112 let file_schema = make_test_schema(&[
1113 Field::new("ts", DataType::Utf8, true),
1114 Field::new("host_id", DataType::UInt8, true),
1115 Field::new("reading_value", DataType::Float64, true),
1116 ]);
1117 let table_schema = make_test_schema(&[
1118 Field::new("host_id", DataType::UInt32, true),
1119 Field::new("reading_value", DataType::Float64, true),
1120 Field::new("ts", DataType::Utf8, true),
1121 ]);
1122
1123 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1124 .unwrap();
1125 }
1126
1127 #[test]
1128 fn test_strict_csv_headers_rejects_unknown_columns() {
1129 let file_schema = make_test_schema(&[
1130 Field::new("host_id", DataType::UInt8, true),
1131 Field::new("reading_value", DataType::Float64, true),
1132 Field::new("ts", DataType::Utf8, true),
1133 Field::new("extra", DataType::Utf8, true),
1134 ]);
1135 let table_schema = make_test_schema(&[
1136 Field::new("host_id", DataType::UInt32, true),
1137 Field::new("reading_value", DataType::Float64, true),
1138 Field::new("ts", DataType::Utf8, true),
1139 ]);
1140
1141 let err =
1142 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1143 .unwrap_err();
1144
1145 assert!(matches!(
1146 err,
1147 error::Error::CsvHeaderMismatch {
1148 unknown_columns,
1149 missing_columns,
1150 duplicate_columns,
1151 ..
1152 } if unknown_columns == vec!["extra".to_string()]
1153 && missing_columns.is_empty()
1154 && duplicate_columns.is_empty()
1155 ));
1156 }
1157
1158 #[test]
1159 fn test_strict_csv_headers_rejects_missing_columns() {
1160 let file_schema = make_test_schema(&[
1161 Field::new("host_id", DataType::UInt8, true),
1162 Field::new("ts", DataType::Utf8, true),
1163 ]);
1164 let table_schema = make_test_schema(&[
1165 Field::new("host_id", DataType::UInt32, true),
1166 Field::new("reading_value", DataType::Float64, true),
1167 Field::new("ts", DataType::Utf8, true),
1168 ]);
1169
1170 let err =
1171 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1172 .unwrap_err();
1173
1174 assert!(matches!(
1175 err,
1176 error::Error::CsvHeaderMismatch {
1177 unknown_columns,
1178 missing_columns,
1179 duplicate_columns,
1180 ..
1181 } if unknown_columns.is_empty()
1182 && missing_columns == vec!["reading_value".to_string()]
1183 && duplicate_columns.is_empty()
1184 ));
1185 }
1186
1187 #[test]
1188 fn test_strict_csv_headers_rejects_duplicate_columns() {
1189 let file_schema = make_test_schema(&[
1190 Field::new("host_id", DataType::UInt8, true),
1191 Field::new("reading_value", DataType::Float64, true),
1192 Field::new("ts", DataType::Utf8, true),
1193 Field::new("host_id", DataType::UInt16, true),
1194 ]);
1195 let table_schema = make_test_schema(&[
1196 Field::new("host_id", DataType::UInt32, true),
1197 Field::new("reading_value", DataType::Float64, true),
1198 Field::new("ts", DataType::Utf8, true),
1199 ]);
1200
1201 let err =
1202 validate_csv_headers_if_required(&make_strict_csv_metadata(file_schema), &table_schema)
1203 .unwrap_err();
1204
1205 assert!(matches!(
1206 err,
1207 error::Error::CsvHeaderMismatch {
1208 unknown_columns,
1209 missing_columns,
1210 duplicate_columns,
1211 ..
1212 } if unknown_columns.is_empty()
1213 && missing_columns.is_empty()
1214 && duplicate_columns == vec!["host_id".to_string()]
1215 ));
1216 }
1217
1218 #[test]
1219 fn test_headerless_csv_schema_projection_is_positional() {
1220 let file_schema = make_test_schema(&[
1221 Field::new("column_1", DataType::UInt8, true),
1222 Field::new("column_2", DataType::Float64, true),
1223 Field::new("column_3", DataType::Utf8, true),
1224 ]);
1225 let table_schema = make_test_schema(&[
1226 Field::new("host_id", DataType::UInt32, true),
1227 Field::new("reading_value", DataType::Float64, true),
1228 Field::new(
1229 "ts",
1230 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1231 true,
1232 ),
1233 ]);
1234
1235 let mapping =
1236 copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1237
1238 assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1239 assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1240 assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1241 assert_field(
1242 &mapping.compat_file_schema,
1243 1,
1244 "reading_value",
1245 &DataType::Float64,
1246 );
1247 assert_field(
1248 &mapping.compat_file_schema,
1249 2,
1250 "ts",
1251 table_schema.field(2).data_type(),
1252 );
1253 assert_eq!(
1254 mapping
1255 .compat_file_schema
1256 .project(&mapping.file_projection)
1257 .unwrap(),
1258 table_schema.project(&mapping.table_projection).unwrap()
1259 );
1260 }
1261
1262 #[test]
1263 fn test_headerless_csv_schema_projection_ignores_extra_file_columns() {
1264 let file_schema = make_test_schema(&[
1265 Field::new("column_1", DataType::UInt8, true),
1266 Field::new("column_2", DataType::Float64, true),
1267 Field::new("column_3", DataType::Utf8, true),
1268 Field::new("column_4", DataType::Utf8, true),
1269 ]);
1270 let table_schema = make_test_schema(&[
1271 Field::new("host_id", DataType::UInt32, true),
1272 Field::new("reading_value", DataType::Float64, true),
1273 Field::new("ts", DataType::Utf8, true),
1274 ]);
1275
1276 let mapping =
1277 copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1278
1279 assert_eq!(mapping.file_projection, vec![0, 1, 2]);
1280 assert_eq!(mapping.table_projection, vec![0, 1, 2]);
1281 assert_eq!(mapping.compat_file_schema.fields().len(), 4);
1282 assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1283 assert_field(
1284 &mapping.compat_file_schema,
1285 1,
1286 "reading_value",
1287 &DataType::Float64,
1288 );
1289 assert_field(&mapping.compat_file_schema, 2, "ts", &DataType::Utf8);
1290 assert_field(&mapping.compat_file_schema, 3, "column_4", &DataType::Utf8);
1291 }
1292
1293 #[test]
1294 fn test_headerless_csv_schema_projection_supports_prefix_import() {
1295 let file_schema = make_test_schema(&[
1296 Field::new("column_1", DataType::UInt8, true),
1297 Field::new("column_2", DataType::Float64, true),
1298 ]);
1299 let table_schema = make_test_schema(&[
1300 Field::new("host_id", DataType::UInt32, true),
1301 Field::new("reading_value", DataType::Float64, true),
1302 Field::new("ts", DataType::Utf8, true),
1303 ]);
1304
1305 let mapping =
1306 copy_from_schema_mapping(&make_csv_metadata(file_schema, false), &table_schema);
1307
1308 assert_eq!(mapping.file_projection, vec![0, 1]);
1309 assert_eq!(mapping.table_projection, vec![0, 1]);
1310 assert_field(&mapping.compat_file_schema, 0, "host_id", &DataType::UInt32);
1311 assert_field(
1312 &mapping.compat_file_schema,
1313 1,
1314 "reading_value",
1315 &DataType::Float64,
1316 );
1317 assert_eq!(
1318 mapping
1319 .compat_file_schema
1320 .project(&mapping.file_projection)
1321 .unwrap(),
1322 table_schema.project(&mapping.table_projection).unwrap()
1323 );
1324 }
1325
1326 #[test]
1327 fn test_csv_reader_schema_for_skip_bad_records_uses_positional_mapping() {
1328 let file_schema = make_test_schema(&[
1329 Field::new("column_1", DataType::Utf8, true),
1330 Field::new("column_2", DataType::Utf8, true),
1331 Field::new("column_3", DataType::Utf8, true),
1332 ]);
1333 let table_schema = make_test_schema(&[
1334 Field::new("host_id", DataType::UInt32, true),
1335 Field::new("jsons", DataType::Binary, true),
1336 Field::new(
1337 "ts",
1338 DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
1339 true,
1340 ),
1341 ]);
1342 let mapping = copy_from_schema_mapping(
1343 &make_csv_metadata(file_schema.clone(), false),
1344 &table_schema,
1345 );
1346 let compat_schema = Arc::new(mapping.compat_file_schema);
1347
1348 let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema);
1349
1350 assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32);
1351 assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8);
1352 assert_eq!(
1353 reader_schema.field(2).data_type(),
1354 table_schema.field(2).data_type()
1355 );
1356 }
1357}