sql/
statements.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod admin;
16pub mod alter;
17pub mod comment;
18pub mod copy;
19pub mod create;
20pub mod cursor;
21pub mod delete;
22pub mod describe;
23pub mod drop;
24pub mod explain;
25pub mod insert;
26pub mod kill;
27mod option_map;
28pub mod query;
29pub mod set_variables;
30pub mod show;
31pub mod statement;
32pub mod tql;
33pub(crate) mod transform;
34pub mod truncate;
35
36use std::sync::Arc;
37
38use api::helper::ColumnDataTypeWrapper;
39use api::v1::SemanticType;
40use common_sql::default_constraint::parse_column_default_constraint;
41use common_time::timezone::Timezone;
42use datatypes::extension::json::{Json2ExtensionType, JsonMetadata};
43use datatypes::json::JsonSettings;
44use datatypes::prelude::ConcreteDataType;
45use datatypes::schema::{COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema};
46use datatypes::types::json_type::JsonNativeType;
47use datatypes::types::{JsonFormat, JsonType, TimestampType};
48use datatypes::value::Value;
49use snafu::ResultExt;
50use sqlparser::ast::{ExactNumberInfo, Ident};
51
52use crate::ast::{
53    ColumnDef, ColumnOption, DataType as SqlDataType, ObjectNamePartExt, TimezoneInfo,
54    Value as SqlValue,
55};
56use crate::error::{
57    self, ConvertToGrpcDataTypeSnafu, ConvertValueSnafu, Result,
58    SerializeColumnDefaultConstraintSnafu, SetFulltextOptionSnafu, SetSkippingIndexOptionSnafu,
59    SetVectorIndexOptionSnafu, SqlCommonSnafu,
60};
61use crate::statements::create::Column;
62pub use crate::statements::option_map::OptionMap;
63pub(crate) use crate::statements::transform::transform_statements;
64
65const VECTOR_TYPE_NAME: &str = "VECTOR";
66const JSON2_TYPE_NAME: &str = "JSON2";
67
68pub fn value_to_sql_value(val: &Value) -> Result<SqlValue> {
69    Ok(match val {
70        Value::Int8(v) => SqlValue::Number(v.to_string(), false),
71        Value::UInt8(v) => SqlValue::Number(v.to_string(), false),
72        Value::Int16(v) => SqlValue::Number(v.to_string(), false),
73        Value::UInt16(v) => SqlValue::Number(v.to_string(), false),
74        Value::Int32(v) => SqlValue::Number(v.to_string(), false),
75        Value::UInt32(v) => SqlValue::Number(v.to_string(), false),
76        Value::Int64(v) => SqlValue::Number(v.to_string(), false),
77        Value::UInt64(v) => SqlValue::Number(v.to_string(), false),
78        Value::Float32(v) => SqlValue::Number(v.to_string(), false),
79        Value::Float64(v) => SqlValue::Number(v.to_string(), false),
80        Value::Boolean(b) => SqlValue::Boolean(*b),
81        Value::Date(d) => SqlValue::SingleQuotedString(d.to_string()),
82        Value::Timestamp(ts) => SqlValue::SingleQuotedString(ts.to_iso8601_string()),
83        Value::String(s) => SqlValue::SingleQuotedString(s.as_utf8().to_string()),
84        Value::Null => SqlValue::Null,
85        // TODO(dennis): supports binary
86        _ => return ConvertValueSnafu { value: val.clone() }.fail(),
87    })
88}
89
90/// Return true when the `ColumnDef` options contain primary key
91pub fn has_primary_key_option(column_def: &ColumnDef) -> bool {
92    column_def
93        .options
94        .iter()
95        .any(|options| matches!(options.option, ColumnOption::PrimaryKey(..)))
96}
97
98/// Create a `ColumnSchema` from `Column`.
99pub fn column_to_schema(
100    column: &Column,
101    time_index: &str,
102    timezone: Option<&Timezone>,
103) -> Result<ColumnSchema> {
104    let is_time_index = column.name().value == time_index;
105
106    let is_nullable = column
107        .options()
108        .iter()
109        .all(|o| !matches!(o.option, ColumnOption::NotNull))
110        && !is_time_index;
111
112    let name = column.name().value.clone();
113    let data_type = sql_data_type_to_concrete_data_type(column.data_type())?;
114    let default_constraint =
115        parse_column_default_constraint(&name, &data_type, column.options(), timezone)
116            .context(SqlCommonSnafu)?;
117
118    let mut column_schema = ColumnSchema::new(name, data_type, is_nullable)
119        .with_time_index(is_time_index)
120        .with_default_constraint(default_constraint)
121        .context(error::InvalidDefaultSnafu {
122            column: &column.name().value,
123        })?;
124
125    if let Some(ColumnOption::Comment(c)) = column.options().iter().find_map(|o| {
126        if matches!(o.option, ColumnOption::Comment(_)) {
127            Some(&o.option)
128        } else {
129            None
130        }
131    }) {
132        let _ = column_schema
133            .mut_metadata()
134            .insert(COMMENT_KEY.to_string(), c.clone());
135    }
136
137    if let Some(options) = column.extensions.build_fulltext_options()? {
138        column_schema = column_schema
139            .with_fulltext_options(options)
140            .context(SetFulltextOptionSnafu)?;
141    }
142
143    if let Some(options) = column.extensions.build_skipping_index_options()? {
144        column_schema = column_schema
145            .with_skipping_options(options)
146            .context(SetSkippingIndexOptionSnafu)?;
147    }
148
149    if let Some(options) = column.extensions.build_vector_index_options()? {
150        column_schema = column_schema
151            .with_vector_index_options(&options)
152            .context(SetVectorIndexOptionSnafu)?;
153    }
154
155    column_schema.set_inverted_index(column.extensions.inverted_index_options.is_some());
156
157    let is_json2_column = if let SqlDataType::Custom(object_name, _) = column.data_type() {
158        object_name
159            .0
160            .first()
161            .map(|x| x.to_string_unquoted().eq_ignore_ascii_case(JSON2_TYPE_NAME))
162            .unwrap_or_default()
163    } else {
164        false
165    };
166    if is_json2_column {
167        let settings = column
168            .extensions
169            .build_json_settings()?
170            .unwrap_or_else(JsonSettings::new_v2);
171        let extension = Json2ExtensionType::new(Arc::new(JsonMetadata::new(settings)));
172        column_schema.with_extension_type(&extension);
173    }
174
175    Ok(column_schema)
176}
177
178/// Convert `ColumnDef` in sqlparser to `ColumnDef` in gRPC proto.
179pub fn sql_column_def_to_grpc_column_def(
180    col: &ColumnDef,
181    timezone: Option<&Timezone>,
182) -> Result<api::v1::ColumnDef> {
183    let name = col.name.value.clone();
184    let data_type = sql_data_type_to_concrete_data_type(&col.data_type)?;
185
186    let is_nullable = col
187        .options
188        .iter()
189        .all(|o| !matches!(o.option, ColumnOption::NotNull));
190
191    let default_constraint =
192        parse_column_default_constraint(&name, &data_type, &col.options, timezone)
193            .context(SqlCommonSnafu)?
194            .map(ColumnDefaultConstraint::try_into) // serialize default constraint to bytes
195            .transpose()
196            .context(SerializeColumnDefaultConstraintSnafu)?;
197    // convert ConcreteDataType to grpc ColumnDataTypeWrapper
198    let (datatype, datatype_ext) = ColumnDataTypeWrapper::try_from(data_type.clone())
199        .context(ConvertToGrpcDataTypeSnafu)?
200        .to_parts();
201
202    let is_primary_key = col
203        .options
204        .iter()
205        .any(|o| matches!(o.option, ColumnOption::PrimaryKey(..)));
206
207    let semantic_type = if is_primary_key {
208        SemanticType::Tag
209    } else {
210        SemanticType::Field
211    };
212
213    Ok(api::v1::ColumnDef {
214        name,
215        data_type: datatype as i32,
216        is_nullable,
217        default_constraint: default_constraint.unwrap_or_default(),
218        semantic_type: semantic_type as _,
219        comment: String::new(),
220        datatype_extension: datatype_ext,
221        options: None,
222    })
223}
224
225pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<ConcreteDataType> {
226    match data_type {
227        SqlDataType::BigInt(_) | SqlDataType::Int64 => Ok(ConcreteDataType::int64_datatype()),
228        SqlDataType::BigIntUnsigned(_) | SqlDataType::UInt64 => {
229            Ok(ConcreteDataType::uint64_datatype())
230        }
231        SqlDataType::Int(_) | SqlDataType::Integer(_) => Ok(ConcreteDataType::int32_datatype()),
232        SqlDataType::IntUnsigned(_) | SqlDataType::UnsignedInteger => {
233            Ok(ConcreteDataType::uint32_datatype())
234        }
235        SqlDataType::SmallInt(_) => Ok(ConcreteDataType::int16_datatype()),
236        SqlDataType::SmallIntUnsigned(_) => Ok(ConcreteDataType::uint16_datatype()),
237        SqlDataType::TinyInt(_) | SqlDataType::Int8(_) => Ok(ConcreteDataType::int8_datatype()),
238        SqlDataType::TinyIntUnsigned(_) | SqlDataType::Int8Unsigned(_) => {
239            Ok(ConcreteDataType::uint8_datatype())
240        }
241        SqlDataType::Char(_)
242        | SqlDataType::Varchar(_)
243        | SqlDataType::Text
244        | SqlDataType::TinyText
245        | SqlDataType::MediumText
246        | SqlDataType::LongText
247        | SqlDataType::String(_) => Ok(ConcreteDataType::string_datatype()),
248        SqlDataType::Float(_) => Ok(ConcreteDataType::float32_datatype()),
249        SqlDataType::Double(_) | SqlDataType::Float64 => Ok(ConcreteDataType::float64_datatype()),
250        SqlDataType::Boolean => Ok(ConcreteDataType::boolean_datatype()),
251        SqlDataType::Date => Ok(ConcreteDataType::date_datatype()),
252        SqlDataType::Binary(_)
253        | SqlDataType::Blob(_)
254        | SqlDataType::Bytea
255        | SqlDataType::Varbinary(_) => Ok(ConcreteDataType::binary_datatype()),
256        SqlDataType::Datetime(_) => Ok(ConcreteDataType::timestamp_microsecond_datatype()),
257        SqlDataType::Timestamp(precision, _) => Ok(precision
258            .as_ref()
259            .map(|v| TimestampType::try_from(*v))
260            .transpose()
261            .map_err(|_| {
262                error::SqlTypeNotSupportedSnafu {
263                    t: data_type.clone(),
264                }
265                .build()
266            })?
267            .map(|t| ConcreteDataType::timestamp_datatype(t.unit()))
268            .unwrap_or(ConcreteDataType::timestamp_millisecond_datatype())),
269        SqlDataType::Interval { .. } => Ok(ConcreteDataType::interval_month_day_nano_datatype()),
270        SqlDataType::Decimal(exact_info) => match exact_info {
271            ExactNumberInfo::None => Ok(ConcreteDataType::decimal128_default_datatype()),
272            // refer to https://dev.mysql.com/doc/refman/8.0/en/fixed-point-types.html
273            // In standard SQL, the syntax DECIMAL(M) is equivalent to DECIMAL(M,0).
274            ExactNumberInfo::Precision(p) => Ok(ConcreteDataType::decimal128_datatype(*p as u8, 0)),
275            ExactNumberInfo::PrecisionAndScale(p, s) => {
276                Ok(ConcreteDataType::decimal128_datatype(*p as u8, *s as i8))
277            }
278        },
279        SqlDataType::JSON => Ok(ConcreteDataType::Json(JsonType::new(JsonFormat::Jsonb))),
280        // Vector type and JSON2 type
281        SqlDataType::Custom(name, args) if name.0.len() == 1 => {
282            let name = name.0[0].to_string_unquoted().to_ascii_uppercase();
283            match name.as_str() {
284                VECTOR_TYPE_NAME if args.len() == 1 => {
285                    let dim = &args[0];
286                    let dim = dim.parse().map_err(|e| {
287                        error::ParseSqlValueSnafu {
288                            msg: format!("Failed to parse vector dimension '{}': {}", dim, e),
289                        }
290                        .build()
291                    })?;
292                    Ok(ConcreteDataType::vector_datatype(dim))
293                }
294                JSON2_TYPE_NAME if args.is_empty() => {
295                    // Currently, JSON2 is not inferred as any native type initially.
296                    // TODO(fys): infer it later from type hints.
297                    let format = JsonFormat::Json2(Arc::new(JsonNativeType::Null));
298                    Ok(ConcreteDataType::Json(JsonType::new(format)))
299                }
300                _ => error::SqlTypeNotSupportedSnafu {
301                    t: data_type.clone(),
302                }
303                .fail(),
304            }
305        }
306        _ => error::SqlTypeNotSupportedSnafu {
307            t: data_type.clone(),
308        }
309        .fail(),
310    }
311}
312
313pub fn concrete_data_type_to_sql_data_type(data_type: &ConcreteDataType) -> Result<SqlDataType> {
314    match data_type {
315        ConcreteDataType::Int64(_) => Ok(SqlDataType::BigInt(None)),
316        ConcreteDataType::UInt64(_) => Ok(SqlDataType::BigIntUnsigned(None)),
317        ConcreteDataType::Int32(_) => Ok(SqlDataType::Int(None)),
318        ConcreteDataType::UInt32(_) => Ok(SqlDataType::IntUnsigned(None)),
319        ConcreteDataType::Int16(_) => Ok(SqlDataType::SmallInt(None)),
320        ConcreteDataType::UInt16(_) => Ok(SqlDataType::SmallIntUnsigned(None)),
321        ConcreteDataType::Int8(_) => Ok(SqlDataType::TinyInt(None)),
322        ConcreteDataType::UInt8(_) => Ok(SqlDataType::TinyIntUnsigned(None)),
323        ConcreteDataType::String(_) => Ok(SqlDataType::String(None)),
324        ConcreteDataType::Float32(_) => Ok(SqlDataType::Float(ExactNumberInfo::None)),
325        ConcreteDataType::Float64(_) => Ok(SqlDataType::Double(ExactNumberInfo::None)),
326        ConcreteDataType::Boolean(_) => Ok(SqlDataType::Boolean),
327        ConcreteDataType::Date(_) => Ok(SqlDataType::Date),
328        ConcreteDataType::Timestamp(ts_type) => Ok(SqlDataType::Timestamp(
329            Some(ts_type.precision()),
330            TimezoneInfo::None,
331        )),
332        ConcreteDataType::Time(time_type) => Ok(SqlDataType::Time(
333            Some(time_type.precision()),
334            TimezoneInfo::None,
335        )),
336        ConcreteDataType::Interval(_) => Ok(SqlDataType::Interval {
337            fields: None,
338            precision: None,
339        }),
340        ConcreteDataType::Binary(_) => Ok(SqlDataType::Varbinary(None)),
341        ConcreteDataType::Decimal128(d) => Ok(SqlDataType::Decimal(
342            ExactNumberInfo::PrecisionAndScale(d.precision() as u64, d.scale() as i64),
343        )),
344        ConcreteDataType::Json(_) => Ok(SqlDataType::JSON),
345        ConcreteDataType::Vector(v) => Ok(SqlDataType::Custom(
346            vec![Ident::new(VECTOR_TYPE_NAME)].into(),
347            vec![v.dim.to_string()],
348        )),
349        ConcreteDataType::Duration(_)
350        | ConcreteDataType::Null(_)
351        | ConcreteDataType::List(_)
352        | ConcreteDataType::Struct(_)
353        | ConcreteDataType::Dictionary(_) => error::ConcreteTypeNotSupportedSnafu {
354            t: data_type.clone(),
355        }
356        .fail(),
357    }
358}
359
360#[cfg(test)]
361mod tests {
362    use api::v1::ColumnDataType;
363    use datatypes::schema::{
364        COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, FulltextAnalyzer,
365    };
366    use sqlparser::ast::{ColumnOptionDef, Expr, PrimaryKeyConstraint};
367
368    use super::*;
369    use crate::ast::TimezoneInfo;
370    use crate::statements::ColumnOption;
371    use crate::statements::create::ColumnExtensions;
372
373    fn check_type(sql_type: SqlDataType, data_type: ConcreteDataType) {
374        assert_eq!(
375            data_type,
376            sql_data_type_to_concrete_data_type(&sql_type).unwrap()
377        );
378    }
379
380    #[test]
381    pub fn test_sql_data_type_to_concrete_data_type() {
382        check_type(
383            SqlDataType::BigInt(None),
384            ConcreteDataType::int64_datatype(),
385        );
386        check_type(SqlDataType::Int(None), ConcreteDataType::int32_datatype());
387        check_type(
388            SqlDataType::Integer(None),
389            ConcreteDataType::int32_datatype(),
390        );
391        check_type(
392            SqlDataType::SmallInt(None),
393            ConcreteDataType::int16_datatype(),
394        );
395        check_type(SqlDataType::Char(None), ConcreteDataType::string_datatype());
396        check_type(
397            SqlDataType::Varchar(None),
398            ConcreteDataType::string_datatype(),
399        );
400        check_type(SqlDataType::Text, ConcreteDataType::string_datatype());
401        check_type(
402            SqlDataType::String(None),
403            ConcreteDataType::string_datatype(),
404        );
405        check_type(
406            SqlDataType::Float(ExactNumberInfo::None),
407            ConcreteDataType::float32_datatype(),
408        );
409        check_type(
410            SqlDataType::Double(ExactNumberInfo::None),
411            ConcreteDataType::float64_datatype(),
412        );
413        check_type(SqlDataType::Boolean, ConcreteDataType::boolean_datatype());
414        check_type(SqlDataType::Date, ConcreteDataType::date_datatype());
415        check_type(
416            SqlDataType::Timestamp(None, TimezoneInfo::None),
417            ConcreteDataType::timestamp_millisecond_datatype(),
418        );
419        check_type(
420            SqlDataType::Varbinary(None),
421            ConcreteDataType::binary_datatype(),
422        );
423        check_type(
424            SqlDataType::BigIntUnsigned(None),
425            ConcreteDataType::uint64_datatype(),
426        );
427        check_type(SqlDataType::UInt64, ConcreteDataType::uint64_datatype());
428        check_type(
429            SqlDataType::IntUnsigned(None),
430            ConcreteDataType::uint32_datatype(),
431        );
432        check_type(
433            SqlDataType::SmallIntUnsigned(None),
434            ConcreteDataType::uint16_datatype(),
435        );
436        check_type(
437            SqlDataType::TinyIntUnsigned(None),
438            ConcreteDataType::uint8_datatype(),
439        );
440        check_type(
441            SqlDataType::Datetime(None),
442            ConcreteDataType::timestamp_microsecond_datatype(),
443        );
444        check_type(
445            SqlDataType::Interval {
446                fields: None,
447                precision: None,
448            },
449            ConcreteDataType::interval_month_day_nano_datatype(),
450        );
451        check_type(SqlDataType::JSON, ConcreteDataType::json_datatype());
452        check_type(
453            SqlDataType::Custom(
454                vec![Ident::new(VECTOR_TYPE_NAME)].into(),
455                vec!["3".to_string()],
456            ),
457            ConcreteDataType::vector_datatype(3),
458        );
459    }
460
461    #[test]
462    pub fn test_sql_column_def_to_grpc_column_def() {
463        // test basic
464        let column_def = ColumnDef {
465            name: "col".into(),
466            data_type: SqlDataType::Double(ExactNumberInfo::None),
467            options: vec![],
468        };
469
470        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
471
472        assert_eq!("col", grpc_column_def.name);
473        assert!(grpc_column_def.is_nullable); // nullable when options are empty
474        assert_eq!(ColumnDataType::Float64 as i32, grpc_column_def.data_type);
475        assert!(grpc_column_def.default_constraint.is_empty());
476        assert_eq!(grpc_column_def.semantic_type, SemanticType::Field as i32);
477
478        // test not null
479        let column_def = ColumnDef {
480            name: "col".into(),
481            data_type: SqlDataType::Double(ExactNumberInfo::None),
482            options: vec![ColumnOptionDef {
483                name: None,
484                option: ColumnOption::NotNull,
485            }],
486        };
487
488        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
489        assert!(!grpc_column_def.is_nullable);
490
491        // test primary key
492        let column_def = ColumnDef {
493            name: "col".into(),
494            data_type: SqlDataType::Double(ExactNumberInfo::None),
495            options: vec![ColumnOptionDef {
496                name: None,
497                option: ColumnOption::PrimaryKey(PrimaryKeyConstraint {
498                    name: None,
499                    index_name: None,
500                    index_type: None,
501                    columns: vec![],
502                    index_options: vec![],
503                    characteristics: None,
504                }),
505            }],
506        };
507
508        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
509        assert_eq!(grpc_column_def.semantic_type, SemanticType::Tag as i32);
510    }
511
512    #[test]
513    pub fn test_sql_column_def_to_grpc_column_def_with_timezone() {
514        let column_def = ColumnDef {
515            name: "col".into(),
516            // MILLISECOND
517            data_type: SqlDataType::Timestamp(Some(3), TimezoneInfo::None),
518            options: vec![ColumnOptionDef {
519                name: None,
520                option: ColumnOption::Default(Expr::Value(
521                    SqlValue::SingleQuotedString("2024-01-30T00:01:01".to_string()).into(),
522                )),
523            }],
524        };
525
526        // with timezone "Asia/Shanghai"
527        let grpc_column_def = sql_column_def_to_grpc_column_def(
528            &column_def,
529            Some(&Timezone::from_tz_string("Asia/Shanghai").unwrap()),
530        )
531        .unwrap();
532        assert_eq!("col", grpc_column_def.name);
533        assert!(grpc_column_def.is_nullable); // nullable when options are empty
534        assert_eq!(
535            ColumnDataType::TimestampMillisecond as i32,
536            grpc_column_def.data_type
537        );
538        assert!(!grpc_column_def.default_constraint.is_empty());
539
540        let constraint =
541            ColumnDefaultConstraint::try_from(&grpc_column_def.default_constraint[..]).unwrap();
542        assert!(
543            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
544                         if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
545        );
546
547        // without timezone
548        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
549        assert_eq!("col", grpc_column_def.name);
550        assert!(grpc_column_def.is_nullable); // nullable when options are empty
551        assert_eq!(
552            ColumnDataType::TimestampMillisecond as i32,
553            grpc_column_def.data_type
554        );
555        assert!(!grpc_column_def.default_constraint.is_empty());
556
557        let constraint =
558            ColumnDefaultConstraint::try_from(&grpc_column_def.default_constraint[..]).unwrap();
559        assert!(
560            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
561                         if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
562        );
563    }
564
565    #[test]
566    pub fn test_has_primary_key_option() {
567        let column_def = ColumnDef {
568            name: "col".into(),
569            data_type: SqlDataType::Double(ExactNumberInfo::None),
570            options: vec![],
571        };
572        assert!(!has_primary_key_option(&column_def));
573
574        let column_def = ColumnDef {
575            name: "col".into(),
576            data_type: SqlDataType::Double(ExactNumberInfo::None),
577            options: vec![ColumnOptionDef {
578                name: None,
579                option: ColumnOption::PrimaryKey(PrimaryKeyConstraint {
580                    name: None,
581                    index_name: None,
582                    index_type: None,
583                    columns: vec![],
584                    index_options: vec![],
585                    characteristics: None,
586                }),
587            }],
588        };
589        assert!(has_primary_key_option(&column_def));
590    }
591
592    #[test]
593    pub fn test_column_to_schema() {
594        let column_def = Column {
595            column_def: ColumnDef {
596                name: "col".into(),
597                data_type: SqlDataType::Double(ExactNumberInfo::None),
598                options: vec![],
599            },
600            extensions: ColumnExtensions::default(),
601        };
602
603        let column_schema = column_to_schema(&column_def, "ts", None).unwrap();
604
605        assert_eq!("col", column_schema.name);
606        assert_eq!(
607            ConcreteDataType::float64_datatype(),
608            column_schema.data_type
609        );
610        assert!(column_schema.is_nullable());
611        assert!(!column_schema.is_time_index());
612
613        let column_schema = column_to_schema(&column_def, "col", None).unwrap();
614
615        assert_eq!("col", column_schema.name);
616        assert_eq!(
617            ConcreteDataType::float64_datatype(),
618            column_schema.data_type
619        );
620        assert!(!column_schema.is_nullable());
621        assert!(column_schema.is_time_index());
622
623        let column_def = Column {
624            column_def: ColumnDef {
625                name: "col2".into(),
626                data_type: SqlDataType::String(None),
627                options: vec![
628                    ColumnOptionDef {
629                        name: None,
630                        option: ColumnOption::NotNull,
631                    },
632                    ColumnOptionDef {
633                        name: None,
634                        option: ColumnOption::Comment("test comment".to_string()),
635                    },
636                ],
637            },
638            extensions: ColumnExtensions::default(),
639        };
640
641        let column_schema = column_to_schema(&column_def, "ts", None).unwrap();
642
643        assert_eq!("col2", column_schema.name);
644        assert_eq!(ConcreteDataType::string_datatype(), column_schema.data_type);
645        assert!(!column_schema.is_nullable());
646        assert!(!column_schema.is_time_index());
647        assert_eq!(
648            column_schema.metadata().get(COMMENT_KEY),
649            Some(&"test comment".to_string())
650        );
651    }
652
653    #[test]
654    fn test_new_json2_column_uses_v2_layout() -> std::result::Result<(), Box<dyn std::error::Error>>
655    {
656        let column = Column {
657            column_def: ColumnDef {
658                name: "data".into(),
659                data_type: SqlDataType::Custom(
660                    sqlparser::ast::ObjectName::from(vec!["JSON2".into()]),
661                    vec![],
662                ),
663                options: vec![],
664            },
665            extensions: ColumnExtensions::default(),
666        };
667
668        let schema = column_to_schema(&column, "ts", None)?;
669        let metadata: serde_json::Value =
670            serde_json::from_str(schema.metadata().get("ARROW:extension:metadata").unwrap())?;
671        assert_eq!(Some(2), metadata["layout_version"].as_u64());
672        assert_eq!(
673            Some(100),
674            metadata["json_settings"]["max_auto_expanded_paths"].as_u64()
675        );
676
677        let mut hinted = column;
678        hinted
679            .extensions
680            .set_json_settings(datatypes::json::JsonSettings::try_new(
681                vec![datatypes::json::JsonTypeHint {
682                    path: vec!["kind".to_string()],
683                    data_type: ConcreteDataType::string_datatype(),
684                    inverted_index: false,
685                }],
686                None,
687            )?)?;
688        let schema = column_to_schema(&hinted, "ts", None)?;
689        let metadata: serde_json::Value =
690            serde_json::from_str(schema.metadata().get("ARROW:extension:metadata").unwrap())?;
691        assert_eq!(
692            Some(100),
693            metadata["json_settings"]["max_auto_expanded_paths"].as_u64()
694        );
695        Ok(())
696    }
697
698    #[test]
699    pub fn test_column_to_schema_timestamp_with_timezone() {
700        let column = Column {
701            column_def: ColumnDef {
702                name: "col".into(),
703                // MILLISECOND
704                data_type: SqlDataType::Timestamp(Some(3), TimezoneInfo::None),
705                options: vec![ColumnOptionDef {
706                    name: None,
707                    option: ColumnOption::Default(Expr::Value(
708                        SqlValue::SingleQuotedString("2024-01-30T00:01:01".to_string()).into(),
709                    )),
710                }],
711            },
712            extensions: ColumnExtensions::default(),
713        };
714
715        // with timezone "Asia/Shanghai"
716
717        let column_schema = column_to_schema(
718            &column,
719            "ts",
720            Some(&Timezone::from_tz_string("Asia/Shanghai").unwrap()),
721        )
722        .unwrap();
723
724        assert_eq!("col", column_schema.name);
725        assert_eq!(
726            ConcreteDataType::timestamp_millisecond_datatype(),
727            column_schema.data_type
728        );
729        assert!(column_schema.is_nullable());
730
731        let constraint = column_schema.default_constraint().unwrap();
732        assert!(
733            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
734                         if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
735        );
736
737        // without timezone
738        let column_schema = column_to_schema(&column, "ts", None).unwrap();
739
740        assert_eq!("col", column_schema.name);
741        assert_eq!(
742            ConcreteDataType::timestamp_millisecond_datatype(),
743            column_schema.data_type
744        );
745        assert!(column_schema.is_nullable());
746
747        let constraint = column_schema.default_constraint().unwrap();
748        assert!(
749            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
750                         if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
751        );
752    }
753
754    #[test]
755    fn test_column_to_schema_with_fulltext() {
756        let column = Column {
757            column_def: ColumnDef {
758                name: "col".into(),
759                data_type: SqlDataType::Text,
760                options: vec![],
761            },
762            extensions: ColumnExtensions {
763                fulltext_index_options: Some(OptionMap::from([
764                    (
765                        COLUMN_FULLTEXT_OPT_KEY_ANALYZER.to_string(),
766                        "English".to_string(),
767                    ),
768                    (
769                        COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE.to_string(),
770                        "true".to_string(),
771                    ),
772                ])),
773                ..Default::default()
774            },
775        };
776
777        let column_schema = column_to_schema(&column, "ts", None).unwrap();
778        assert_eq!("col", column_schema.name);
779        assert_eq!(ConcreteDataType::string_datatype(), column_schema.data_type);
780        let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
781        assert_eq!(fulltext_options.analyzer, FulltextAnalyzer::English);
782        assert!(fulltext_options.case_sensitive);
783    }
784
785    #[test]
786    fn test_column_to_schema_with_vector_index() {
787        use datatypes::schema::{VectorDistanceMetric, VectorIndexEngineType};
788
789        // Test with custom metric and parameters
790        let column = Column {
791            column_def: ColumnDef {
792                name: "embedding".into(),
793                data_type: SqlDataType::Custom(
794                    vec![Ident::new(VECTOR_TYPE_NAME)].into(),
795                    vec!["128".to_string()],
796                ),
797                options: vec![],
798            },
799            extensions: ColumnExtensions {
800                vector_index_options: Some(OptionMap::from([
801                    ("metric".to_string(), "cosine".to_string()),
802                    ("connectivity".to_string(), "32".to_string()),
803                    ("expansion_add".to_string(), "200".to_string()),
804                    ("expansion_search".to_string(), "100".to_string()),
805                ])),
806                ..Default::default()
807            },
808        };
809
810        let column_schema = column_to_schema(&column, "ts", None).unwrap();
811        assert_eq!("embedding", column_schema.name);
812        assert!(column_schema.is_vector_indexed());
813
814        let vector_options = column_schema.vector_index_options().unwrap().unwrap();
815        assert_eq!(vector_options.engine, VectorIndexEngineType::Usearch);
816        assert_eq!(vector_options.metric, VectorDistanceMetric::Cosine);
817        assert_eq!(vector_options.connectivity, 32);
818        assert_eq!(vector_options.expansion_add, 200);
819        assert_eq!(vector_options.expansion_search, 100);
820    }
821
822    #[test]
823    fn test_column_to_schema_with_vector_index_defaults() {
824        use datatypes::schema::{VectorDistanceMetric, VectorIndexEngineType};
825
826        // Test with default values (empty options map)
827        let column = Column {
828            column_def: ColumnDef {
829                name: "vec".into(),
830                data_type: SqlDataType::Custom(
831                    vec![Ident::new(VECTOR_TYPE_NAME)].into(),
832                    vec!["64".to_string()],
833                ),
834                options: vec![],
835            },
836            extensions: ColumnExtensions {
837                vector_index_options: Some(OptionMap::default()),
838                ..Default::default()
839            },
840        };
841
842        let column_schema = column_to_schema(&column, "ts", None).unwrap();
843        assert_eq!("vec", column_schema.name);
844        assert!(column_schema.is_vector_indexed());
845
846        let vector_options = column_schema.vector_index_options().unwrap().unwrap();
847        // Verify defaults
848        assert_eq!(vector_options.engine, VectorIndexEngineType::Usearch);
849        assert_eq!(vector_options.metric, VectorDistanceMetric::L2sq);
850        assert_eq!(vector_options.connectivity, 16);
851        assert_eq!(vector_options.expansion_add, 128);
852        assert_eq!(vector_options.expansion_search, 64);
853    }
854}
sql/statements.rs

sql/
statements.rs
