common_sql/
convert.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::str::FromStr;
16
17use arrow_schema::extension::ExtensionType;
18use common_time::Timestamp;
19use common_time::timezone::Timezone;
20use datatypes::extension::json::{Json2ExtensionType, parse_legacy_json2_settings};
21use datatypes::prelude::ConcreteDataType;
22use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
23use datatypes::types::{JsonFormat, parse_string_to_jsonb, parse_string_to_vector_type_value};
24use datatypes::value::{OrderedF32, OrderedF64, Value};
25use snafu::{OptionExt, ResultExt, ensure};
26pub use sqlparser::ast::{
27    BinaryOperator, ColumnDef, ColumnOption, ColumnOptionDef, DataType, Expr, Function,
28    FunctionArg, FunctionArgExpr, FunctionArguments, Ident, ObjectName, SqlOption, TableConstraint,
29    TimezoneInfo, UnaryOperator, Value as SqlValue, Visit, VisitMut, Visitor, VisitorMut,
30    visit_expressions_mut, visit_statements_mut,
31};
32
33use crate::error::{
34    ColumnTypeMismatchSnafu, ConvertSqlValueSnafu, ConvertStrSnafu, DatatypeSnafu,
35    DeserializeSnafu, InvalidCastSnafu, InvalidSqlValueSnafu, InvalidUnaryOpSnafu,
36    ParseSqlValueSnafu, Result, TimestampOverflowSnafu, UnsupportedUnaryOpSnafu,
37};
38
39fn parse_sql_number<R: FromStr + std::fmt::Debug>(n: &str) -> Result<R>
40where
41    <R as FromStr>::Err: std::fmt::Debug,
42{
43    match n.parse::<R>() {
44        Ok(n) => Ok(n),
45        Err(e) => ParseSqlValueSnafu {
46            msg: format!("Fail to parse number {n}, {e:?}"),
47        }
48        .fail(),
49    }
50}
51
52macro_rules! parse_number_to_value {
53    ($data_type: expr, $n: ident,  $(($Type: ident, $PrimitiveType: ident, $Target: ident)), +) => {
54        match $data_type {
55            $(
56                ConcreteDataType::$Type(_) => {
57                    let n  = parse_sql_number::<$PrimitiveType>($n)?;
58                    Ok(Value::$Type($Target::from(n)))
59                },
60            )+
61            ConcreteDataType::Timestamp(t) => {
62                let n = parse_sql_number::<i64>($n)?;
63                let timestamp = Timestamp::new(n, t.unit());
64
65                // Check if the value is within the valid range for the target unit
66                if Timestamp::is_overflow(n, t.unit()) {
67                    return TimestampOverflowSnafu {
68                        timestamp,
69                        target_unit: t.unit(),
70                    }.fail();
71                }
72
73                Ok(Value::Timestamp(timestamp))
74            },
75            // TODO(QuenKar): This could need to be optimized
76            // if this from_str function is slow,
77            // we can implement parse decimal string with precision and scale manually.
78            ConcreteDataType::Decimal128(_) => {
79                if let Ok(val) = common_decimal::Decimal128::from_str($n) {
80                    Ok(Value::Decimal128(val))
81                } else {
82                    ParseSqlValueSnafu {
83                        msg: format!("Fail to parse number {}, invalid column type: {:?}",
84                                        $n, $data_type)
85                    }.fail()
86                }
87            }
88            // It's valid for MySQL JDBC to send "0" and "1" for boolean types, so adapt to that.
89            ConcreteDataType::Boolean(_) => {
90                match $n {
91                    "0" => Ok(Value::Boolean(false)),
92                    "1" => Ok(Value::Boolean(true)),
93                    _ => ParseSqlValueSnafu {
94                        msg: format!("Failed to parse number '{}' to boolean column type", $n)}.fail(),
95                }
96            }
97            _ => ParseSqlValueSnafu {
98                msg: format!("Fail to parse number {}, invalid column type: {:?}",
99                                $n, $data_type
100                )}.fail(),
101        }
102    }
103}
104
105/// Convert a sql value into datatype's value
106pub(crate) fn sql_number_to_value(data_type: &ConcreteDataType, n: &str) -> Result<Value> {
107    parse_number_to_value!(
108        data_type,
109        n,
110        (UInt8, u8, u8),
111        (UInt16, u16, u16),
112        (UInt32, u32, u32),
113        (UInt64, u64, u64),
114        (Int8, i8, i8),
115        (Int16, i16, i16),
116        (Int32, i32, i32),
117        (Int64, i64, i64),
118        (Float64, f64, OrderedF64),
119        (Float32, f32, OrderedF32)
120    )
121    // TODO(hl): also Date/DateTime
122}
123
124/// Converts SQL value to value according to the data type.
125/// If `auto_string_to_numeric` is true, tries to cast the string value to numeric values,
126/// and returns error if the cast fails.
127pub fn sql_value_to_value(
128    column_schema: &ColumnSchema,
129    sql_val: &SqlValue,
130    timezone: Option<&Timezone>,
131    mut unary_op: Option<UnaryOperator>,
132    auto_string_to_numeric: bool,
133) -> Result<Value> {
134    let column_name = &column_schema.name;
135    let data_type = &column_schema.data_type;
136    let mut value = match sql_val {
137        // Parse the sign with the digits, otherwise the minimum of a signed integer
138        // or timestamp overflows before it is negated. Other types keep the negation
139        // below: it accepts unsigned `-0`, and the decimal parser depends on length.
140        SqlValue::Number(n, _)
141            if unary_op == Some(UnaryOperator::Minus)
142                && matches!(
143                    data_type,
144                    ConcreteDataType::Int8(_)
145                        | ConcreteDataType::Int16(_)
146                        | ConcreteDataType::Int32(_)
147                        | ConcreteDataType::Int64(_)
148                        | ConcreteDataType::Timestamp(_)
149                ) =>
150        {
151            unary_op = None;
152            sql_number_to_value(data_type, &format!("-{n}"))?
153        }
154        SqlValue::Number(n, _) => sql_number_to_value(data_type, n)?,
155        SqlValue::Null => Value::Null,
156        SqlValue::Boolean(b) => {
157            ensure!(
158                data_type.is_boolean(),
159                ColumnTypeMismatchSnafu {
160                    column_name,
161                    expect: data_type.clone(),
162                    actual: ConcreteDataType::boolean_datatype(),
163                }
164            );
165
166            (*b).into()
167        }
168        SqlValue::DoubleQuotedString(s) | SqlValue::SingleQuotedString(s) => {
169            parse_string_to_value(column_schema, s.clone(), timezone, auto_string_to_numeric)?
170        }
171        SqlValue::HexStringLiteral(s) => {
172            // Should not directly write binary into json column
173            ensure!(
174                !matches!(data_type, ConcreteDataType::Json(_)),
175                ColumnTypeMismatchSnafu {
176                    column_name,
177                    expect: ConcreteDataType::binary_datatype(),
178                    actual: ConcreteDataType::json_datatype(),
179                }
180            );
181
182            parse_hex_string(s)?
183        }
184        SqlValue::Placeholder(s) => return InvalidSqlValueSnafu { value: s }.fail(),
185
186        // TODO(dennis): supports binary string
187        _ => {
188            return ConvertSqlValueSnafu {
189                value: sql_val.clone(),
190                datatype: data_type.clone(),
191            }
192            .fail();
193        }
194    };
195
196    if let Some(unary_op) = unary_op {
197        match unary_op {
198            UnaryOperator::Plus | UnaryOperator::Minus | UnaryOperator::Not => {}
199            _ => {
200                return UnsupportedUnaryOpSnafu { unary_op }.fail();
201            }
202        }
203
204        match value {
205            Value::Null => {}
206            Value::Boolean(bool) => match unary_op {
207                UnaryOperator::Not => value = Value::Boolean(!bool),
208                _ => {
209                    return InvalidUnaryOpSnafu { unary_op, value }.fail();
210                }
211            },
212            Value::UInt8(_)
213            | Value::UInt16(_)
214            | Value::UInt32(_)
215            | Value::UInt64(_)
216            | Value::Int8(_)
217            | Value::Int16(_)
218            | Value::Int32(_)
219            | Value::Int64(_)
220            | Value::Float32(_)
221            | Value::Float64(_)
222            | Value::Decimal128(_)
223            | Value::Date(_)
224            | Value::Timestamp(_)
225            | Value::Time(_)
226            | Value::Duration(_)
227            | Value::IntervalYearMonth(_)
228            | Value::IntervalDayTime(_)
229            | Value::IntervalMonthDayNano(_) => match unary_op {
230                UnaryOperator::Plus => {}
231                UnaryOperator::Minus => {
232                    value = value
233                        .try_negative()
234                        .with_context(|| InvalidUnaryOpSnafu { unary_op, value })?;
235                }
236                _ => return InvalidUnaryOpSnafu { unary_op, value }.fail(),
237            },
238
239            Value::String(_)
240            | Value::Binary(_)
241            | Value::List(_)
242            | Value::Struct(_)
243            | Value::Json(_) => {
244                return InvalidUnaryOpSnafu { unary_op, value }.fail();
245            }
246        }
247    }
248
249    let value_datatype = value.data_type();
250    // The datatype of json value is determined by its actual data, so we can't simply "cast" it here.
251    if value_datatype.is_json() || value_datatype == *data_type {
252        Ok(value)
253    } else {
254        datatypes::types::cast(value, data_type).with_context(|_| InvalidCastSnafu {
255            sql_value: sql_val.clone(),
256            datatype: data_type,
257        })
258    }
259}
260
261pub(crate) fn parse_string_to_value(
262    column_schema: &ColumnSchema,
263    s: String,
264    timezone: Option<&Timezone>,
265    auto_string_to_numeric: bool,
266) -> Result<Value> {
267    let data_type = &column_schema.data_type;
268    if auto_string_to_numeric && let Some(value) = auto_cast_to_numeric(&s, data_type)? {
269        return Ok(value);
270    }
271
272    ensure!(
273        data_type.is_stringifiable(),
274        ColumnTypeMismatchSnafu {
275            column_name: column_schema.name.clone(),
276            expect: data_type.clone(),
277            actual: ConcreteDataType::string_datatype(),
278        }
279    );
280
281    match data_type {
282        ConcreteDataType::String(_) => Ok(Value::String(s.into())),
283        ConcreteDataType::Date(_) => {
284            if let Ok(date) = common_time::date::Date::from_str(&s, timezone) {
285                Ok(Value::Date(date))
286            } else {
287                ParseSqlValueSnafu {
288                    msg: format!("Failed to parse {s} to Date value"),
289                }
290                .fail()
291            }
292        }
293        ConcreteDataType::Timestamp(t) => {
294            if let Ok(ts) = Timestamp::from_str(&s, timezone) {
295                Ok(Value::Timestamp(ts.convert_to(t.unit()).context(
296                    TimestampOverflowSnafu {
297                        timestamp: ts,
298                        target_unit: t.unit(),
299                    },
300                )?))
301            } else if let Ok(ts) = i64::from_str(s.as_str()) {
302                Ok(Value::Timestamp(Timestamp::new(ts, t.unit())))
303            } else {
304                ParseSqlValueSnafu {
305                    msg: format!("Failed to parse {s} to Timestamp value"),
306                }
307                .fail()
308            }
309        }
310        ConcreteDataType::Decimal128(_) => {
311            if let Ok(val) = common_decimal::Decimal128::from_str(&s) {
312                Ok(Value::Decimal128(val))
313            } else {
314                ParseSqlValueSnafu {
315                    msg: format!("Fail to parse number {s} to Decimal128 value"),
316                }
317                .fail()
318            }
319        }
320        ConcreteDataType::Binary(_) => Ok(Value::Binary(s.as_bytes().into())),
321        ConcreteDataType::Json(j) => match &j.format {
322            JsonFormat::Jsonb => {
323                let v = parse_string_to_jsonb(&s).context(DatatypeSnafu)?;
324                Ok(Value::Binary(v.into()))
325            }
326            JsonFormat::Json2(_) => {
327                let v = serde_json::from_str(&s).context(DeserializeSnafu { json: s })?;
328
329                if let Some(extension) = column_schema
330                    .extension_type::<Json2ExtensionType>()
331                    .context(DatatypeSnafu)?
332                {
333                    extension.metadata().json_settings().encode(v)
334                } else {
335                    parse_legacy_json2_settings(column_schema.metadata())
336                        .context(DatatypeSnafu)?
337                        .unwrap_or_default()
338                        .encode(v)
339                }
340                .context(DatatypeSnafu)
341            }
342        },
343        ConcreteDataType::Vector(d) => {
344            let v = parse_string_to_vector_type_value(&s, Some(d.dim)).context(DatatypeSnafu)?;
345            Ok(Value::Binary(v.into()))
346        }
347        _ => ParseSqlValueSnafu {
348            msg: format!("Failed to parse {s} to {data_type} value"),
349        }
350        .fail(),
351    }
352}
353
354/// Casts string to value of specified numeric data type.
355/// If the string cannot be parsed, returns an error.
356///
357/// Returns None if the data type doesn't support auto casting.
358pub(crate) fn auto_cast_to_numeric(s: &str, data_type: &ConcreteDataType) -> Result<Option<Value>> {
359    let value = match data_type {
360        ConcreteDataType::Boolean(_) => s.parse::<bool>().map(Value::Boolean).ok(),
361        ConcreteDataType::Int8(_) => s.parse::<i8>().map(Value::Int8).ok(),
362        ConcreteDataType::Int16(_) => s.parse::<i16>().map(Value::Int16).ok(),
363        ConcreteDataType::Int32(_) => s.parse::<i32>().map(Value::Int32).ok(),
364        ConcreteDataType::Int64(_) => s.parse::<i64>().map(Value::Int64).ok(),
365        ConcreteDataType::UInt8(_) => s.parse::<u8>().map(Value::UInt8).ok(),
366        ConcreteDataType::UInt16(_) => s.parse::<u16>().map(Value::UInt16).ok(),
367        ConcreteDataType::UInt32(_) => s.parse::<u32>().map(Value::UInt32).ok(),
368        ConcreteDataType::UInt64(_) => s.parse::<u64>().map(Value::UInt64).ok(),
369        ConcreteDataType::Float32(_) => s
370            .parse::<f32>()
371            .map(|v| Value::Float32(OrderedF32::from(v)))
372            .ok(),
373        ConcreteDataType::Float64(_) => s
374            .parse::<f64>()
375            .map(|v| Value::Float64(OrderedF64::from(v)))
376            .ok(),
377        _ => return Ok(None),
378    };
379
380    match value {
381        Some(value) => Ok(Some(value)),
382        None => ConvertStrSnafu {
383            value: s,
384            datatype: data_type.clone(),
385        }
386        .fail(),
387    }
388}
389
390pub(crate) fn parse_hex_string(s: &str) -> Result<Value> {
391    match hex::decode(s) {
392        Ok(b) => Ok(Value::Binary(common_base::bytes::Bytes::from(b))),
393        Err(hex::FromHexError::InvalidHexCharacter { c, index }) => ParseSqlValueSnafu {
394            msg: format!(
395                "Fail to parse hex string to Byte: invalid character {c:?} at position {index}"
396            ),
397        }
398        .fail(),
399        Err(hex::FromHexError::OddLength) => ParseSqlValueSnafu {
400            msg: "Fail to parse hex string to Byte: odd number of digits".to_string(),
401        }
402        .fail(),
403        Err(e) => ParseSqlValueSnafu {
404            msg: format!("Fail to parse hex string to Byte {s}, {e:?}"),
405        }
406        .fail(),
407    }
408}
409
410/// Deserialize default constraint from json bytes
411pub fn deserialize_default_constraint(
412    bytes: &[u8],
413    column_name: &str,
414    data_type: &ConcreteDataType,
415) -> Result<Option<ColumnDefaultConstraint>> {
416    let json = String::from_utf8_lossy(bytes);
417    let default_constraint = serde_json::from_str(&json).context(DeserializeSnafu { json })?;
418    let column_def = sqlparser::ast::ColumnOptionDef {
419        name: None,
420        option: sqlparser::ast::ColumnOption::Default(default_constraint),
421    };
422
423    crate::default_constraint::parse_column_default_constraint(
424        column_name,
425        data_type,
426        &[column_def],
427        None,
428    )
429}
430
431#[cfg(test)]
432mod test {
433    use common_base::bytes::Bytes;
434    use common_time::timestamp::TimeUnit;
435    use datatypes::types::TimestampType;
436    use datatypes::value::OrderedFloat;
437
438    use super::*;
439
440    macro_rules! call_parse_string_to_value {
441        ($column_name: expr, $input: expr, $data_type: expr) => {
442            call_parse_string_to_value!($column_name, $input, $data_type, None)
443        };
444        ($column_name: expr, $input: expr, $data_type: expr, timezone = $timezone: expr) => {
445            call_parse_string_to_value!($column_name, $input, $data_type, Some($timezone))
446        };
447        ($column_name: expr, $input: expr, $data_type: expr, $timezone: expr) => {{
448            let column_schema = ColumnSchema::new($column_name, $data_type, true);
449            parse_string_to_value(&column_schema, $input, $timezone, true)
450        }};
451    }
452
453    #[test]
454    fn test_string_to_value_auto_numeric() -> Result<()> {
455        // Test string to boolean with auto cast
456        let result = call_parse_string_to_value!(
457            "col",
458            "true".to_string(),
459            ConcreteDataType::boolean_datatype()
460        )?;
461        assert_eq!(Value::Boolean(true), result);
462
463        // Test invalid string to boolean with auto cast
464        let result = call_parse_string_to_value!(
465            "col",
466            "not_a_boolean".to_string(),
467            ConcreteDataType::boolean_datatype()
468        );
469        assert!(result.is_err());
470
471        // Test string to int8
472        let result = call_parse_string_to_value!(
473            "col",
474            "42".to_string(),
475            ConcreteDataType::int8_datatype()
476        )?;
477        assert_eq!(Value::Int8(42), result);
478
479        // Test invalid string to int8 with auto cast
480        let result = call_parse_string_to_value!(
481            "col",
482            "not_an_int8".to_string(),
483            ConcreteDataType::int8_datatype()
484        );
485        assert!(result.is_err());
486
487        // Test string to int16
488        let result = call_parse_string_to_value!(
489            "col",
490            "1000".to_string(),
491            ConcreteDataType::int16_datatype()
492        )?;
493        assert_eq!(Value::Int16(1000), result);
494
495        // Test invalid string to int16 with auto cast
496        let result = call_parse_string_to_value!(
497            "col",
498            "not_an_int16".to_string(),
499            ConcreteDataType::int16_datatype()
500        );
501        assert!(result.is_err());
502
503        // Test string to int32
504        let result = call_parse_string_to_value!(
505            "col",
506            "100000".to_string(),
507            ConcreteDataType::int32_datatype()
508        )?;
509        assert_eq!(Value::Int32(100000), result);
510
511        // Test invalid string to int32 with auto cast
512        let result = call_parse_string_to_value!(
513            "col",
514            "not_an_int32".to_string(),
515            ConcreteDataType::int32_datatype()
516        );
517        assert!(result.is_err());
518
519        // Test string to int64
520        let result = call_parse_string_to_value!(
521            "col",
522            "1000000".to_string(),
523            ConcreteDataType::int64_datatype()
524        )?;
525        assert_eq!(Value::Int64(1000000), result);
526
527        // Test invalid string to int64 with auto cast
528        let result = call_parse_string_to_value!(
529            "col",
530            "not_an_int64".to_string(),
531            ConcreteDataType::int64_datatype()
532        );
533        assert!(result.is_err());
534
535        // Test string to uint8
536        let result = call_parse_string_to_value!(
537            "col",
538            "200".to_string(),
539            ConcreteDataType::uint8_datatype()
540        )?;
541        assert_eq!(Value::UInt8(200), result);
542
543        // Test invalid string to uint8 with auto cast
544        let result = call_parse_string_to_value!(
545            "col",
546            "not_a_uint8".to_string(),
547            ConcreteDataType::uint8_datatype()
548        );
549        assert!(result.is_err());
550
551        // Test string to uint16
552        let result = call_parse_string_to_value!(
553            "col",
554            "60000".to_string(),
555            ConcreteDataType::uint16_datatype()
556        )?;
557        assert_eq!(Value::UInt16(60000), result);
558
559        // Test invalid string to uint16 with auto cast
560        let result = call_parse_string_to_value!(
561            "col",
562            "not_a_uint16".to_string(),
563            ConcreteDataType::uint16_datatype()
564        );
565        assert!(result.is_err());
566
567        // Test string to uint32
568        let result = call_parse_string_to_value!(
569            "col",
570            "4000000000".to_string(),
571            ConcreteDataType::uint32_datatype()
572        )?;
573        assert_eq!(Value::UInt32(4000000000), result);
574
575        // Test invalid string to uint32 with auto cast
576        let result = call_parse_string_to_value!(
577            "col",
578            "not_a_uint32".to_string(),
579            ConcreteDataType::uint32_datatype()
580        );
581        assert!(result.is_err());
582
583        // Test string to uint64
584        let result = call_parse_string_to_value!(
585            "col",
586            "18446744073709551615".to_string(),
587            ConcreteDataType::uint64_datatype()
588        )?;
589        assert_eq!(Value::UInt64(18446744073709551615), result);
590
591        // Test invalid string to uint64 with auto cast
592        let result = call_parse_string_to_value!(
593            "col",
594            "not_a_uint64".to_string(),
595            ConcreteDataType::uint64_datatype()
596        );
597        assert!(result.is_err());
598
599        // Test string to float32
600        let result = call_parse_string_to_value!(
601            "col",
602            "3.5".to_string(),
603            ConcreteDataType::float32_datatype()
604        )?;
605        assert_eq!(Value::Float32(OrderedF32::from(3.5)), result);
606
607        // Test invalid string to float32 with auto cast
608        let result = call_parse_string_to_value!(
609            "col",
610            "not_a_float32".to_string(),
611            ConcreteDataType::float32_datatype()
612        );
613        assert!(result.is_err());
614
615        // Test string to float64
616        let result = call_parse_string_to_value!(
617            "col",
618            "3.5".to_string(),
619            ConcreteDataType::float64_datatype()
620        )?;
621        assert_eq!(Value::Float64(OrderedF64::from(3.5)), result);
622
623        // Test invalid string to float64 with auto cast
624        let result = call_parse_string_to_value!(
625            "col",
626            "not_a_float64".to_string(),
627            ConcreteDataType::float64_datatype()
628        );
629        assert!(result.is_err());
630        Ok(())
631    }
632
633    macro_rules! call_sql_value_to_value {
634        ($column_name: expr, $data_type: expr, $sql_value: expr) => {
635            call_sql_value_to_value!($column_name, $data_type, $sql_value, None, None, false)
636        };
637        ($column_name: expr, $data_type: expr, $sql_value: expr, timezone = $timezone: expr) => {
638            call_sql_value_to_value!(
639                $column_name,
640                $data_type,
641                $sql_value,
642                Some($timezone),
643                None,
644                false
645            )
646        };
647        ($column_name: expr, $data_type: expr, $sql_value: expr, unary_op = $unary_op: expr) => {
648            call_sql_value_to_value!(
649                $column_name,
650                $data_type,
651                $sql_value,
652                None,
653                Some($unary_op),
654                false
655            )
656        };
657        ($column_name: expr, $data_type: expr, $sql_value: expr, auto_string_to_numeric) => {
658            call_sql_value_to_value!($column_name, $data_type, $sql_value, None, None, true)
659        };
660        ($column_name: expr, $data_type: expr, $sql_value: expr, $timezone: expr, $unary_op: expr, $auto_string_to_numeric: expr) => {{
661            let column_schema = ColumnSchema::new($column_name, $data_type, true);
662            sql_value_to_value(
663                &column_schema,
664                $sql_value,
665                $timezone,
666                $unary_op,
667                $auto_string_to_numeric,
668            )
669        }};
670    }
671
672    #[test]
673    fn test_sql_value_to_value() -> Result<()> {
674        let sql_val = SqlValue::Null;
675        assert_eq!(
676            Value::Null,
677            call_sql_value_to_value!("a", ConcreteDataType::float64_datatype(), &sql_val)?
678        );
679
680        let sql_val = SqlValue::Boolean(true);
681        assert_eq!(
682            Value::Boolean(true),
683            call_sql_value_to_value!("a", ConcreteDataType::boolean_datatype(), &sql_val)?
684        );
685
686        let sql_val = SqlValue::Number("3.0".to_string(), false);
687        assert_eq!(
688            Value::Float64(OrderedFloat(3.0)),
689            call_sql_value_to_value!("a", ConcreteDataType::float64_datatype(), &sql_val)?
690        );
691
692        let sql_val = SqlValue::Number("3.0".to_string(), false);
693        let v = call_sql_value_to_value!("a", ConcreteDataType::boolean_datatype(), &sql_val);
694        assert!(v.is_err());
695        assert!(format!("{v:?}").contains("Failed to parse number '3.0' to boolean column type"));
696
697        let sql_val = SqlValue::Boolean(true);
698        let v = call_sql_value_to_value!("a", ConcreteDataType::float64_datatype(), &sql_val);
699        assert!(v.is_err());
700        assert!(
701            format!("{v:?}").contains(
702                "Column a expect type: Float64(Float64Type), actual: Boolean(BooleanType)"
703            ),
704            "v is {v:?}",
705        );
706
707        let sql_val = SqlValue::HexStringLiteral("48656c6c6f20776f726c6421".to_string());
708        let v = call_sql_value_to_value!("a", ConcreteDataType::binary_datatype(), &sql_val)?;
709        assert_eq!(Value::Binary(Bytes::from(b"Hello world!".as_slice())), v);
710
711        let sql_val = SqlValue::DoubleQuotedString("MorningMyFriends".to_string());
712        let v = call_sql_value_to_value!("a", ConcreteDataType::binary_datatype(), &sql_val)?;
713        assert_eq!(
714            Value::Binary(Bytes::from(b"MorningMyFriends".as_slice())),
715            v
716        );
717
718        let sql_val = SqlValue::HexStringLiteral("9AF".to_string());
719        let v = call_sql_value_to_value!("a", ConcreteDataType::binary_datatype(), &sql_val);
720        assert!(v.is_err());
721        assert!(
722            format!("{v:?}").contains("odd number of digits"),
723            "v is {v:?}"
724        );
725
726        let sql_val = SqlValue::HexStringLiteral("AG".to_string());
727        let v = call_sql_value_to_value!("a", ConcreteDataType::binary_datatype(), &sql_val);
728        assert!(v.is_err());
729        assert!(format!("{v:?}").contains("invalid character"), "v is {v:?}",);
730
731        let sql_val = SqlValue::DoubleQuotedString("MorningMyFriends".to_string());
732        let v = call_sql_value_to_value!("a", ConcreteDataType::json_datatype(), &sql_val);
733        assert!(v.is_err());
734
735        let sql_val = SqlValue::DoubleQuotedString(r#"{"a":"b"}"#.to_string());
736        let v = call_sql_value_to_value!("a", ConcreteDataType::json_datatype(), &sql_val)?;
737        assert_eq!(
738            Value::Binary(Bytes::from(
739                jsonb::parse_value(r#"{"a":"b"}"#.as_bytes())
740                    .unwrap()
741                    .to_vec()
742                    .as_slice()
743            )),
744            v
745        );
746        Ok(())
747    }
748
749    #[test]
750    fn test_parse_json_to_jsonb() {
751        match call_parse_string_to_value!(
752            "json_col",
753            r#"{"a": "b"}"#.to_string(),
754            ConcreteDataType::json_datatype()
755        ) {
756            Ok(Value::Binary(b)) => {
757                assert_eq!(
758                    b,
759                    jsonb::parse_value(r#"{"a": "b"}"#.as_bytes())
760                        .unwrap()
761                        .to_vec()
762                );
763            }
764            _ => {
765                unreachable!()
766            }
767        }
768
769        assert!(
770            call_parse_string_to_value!(
771                "json_col",
772                r#"Nicola Kovac is the best rifler in the world"#.to_string(),
773                ConcreteDataType::json_datatype()
774            )
775            .is_err()
776        )
777    }
778
779    #[test]
780    fn test_sql_number_to_value() {
781        let v = sql_number_to_value(&ConcreteDataType::float64_datatype(), "3.0").unwrap();
782        assert_eq!(Value::Float64(OrderedFloat(3.0)), v);
783
784        let v = sql_number_to_value(&ConcreteDataType::int32_datatype(), "999").unwrap();
785        assert_eq!(Value::Int32(999), v);
786
787        let v = sql_number_to_value(
788            &ConcreteDataType::timestamp_nanosecond_datatype(),
789            "1073741821",
790        )
791        .unwrap();
792        assert_eq!(Value::Timestamp(Timestamp::new_nanosecond(1073741821)), v);
793
794        let v = sql_number_to_value(
795            &ConcreteDataType::timestamp_millisecond_datatype(),
796            "999999",
797        )
798        .unwrap();
799        assert_eq!(Value::Timestamp(Timestamp::new_millisecond(999999)), v);
800
801        let v = sql_number_to_value(&ConcreteDataType::string_datatype(), "999");
802        assert!(v.is_err(), "parse value error is: {v:?}");
803
804        let v = sql_number_to_value(&ConcreteDataType::boolean_datatype(), "0").unwrap();
805        assert_eq!(v, Value::Boolean(false));
806        let v = sql_number_to_value(&ConcreteDataType::boolean_datatype(), "1").unwrap();
807        assert_eq!(v, Value::Boolean(true));
808        assert!(sql_number_to_value(&ConcreteDataType::boolean_datatype(), "2").is_err());
809    }
810
811    #[test]
812    fn test_parse_date_literal() {
813        let value = call_sql_value_to_value!(
814            "date",
815            ConcreteDataType::date_datatype(),
816            &SqlValue::DoubleQuotedString("2022-02-22".to_string())
817        )
818        .unwrap();
819        assert_eq!(ConcreteDataType::date_datatype(), value.data_type());
820        if let Value::Date(d) = value {
821            assert_eq!("2022-02-22", d.to_string());
822        } else {
823            unreachable!()
824        }
825
826        // with timezone
827        let value = call_sql_value_to_value!(
828            "date",
829            ConcreteDataType::date_datatype(),
830            &SqlValue::DoubleQuotedString("2022-02-22".to_string()),
831            timezone = &Timezone::from_tz_string("+07:00").unwrap()
832        )
833        .unwrap();
834        assert_eq!(ConcreteDataType::date_datatype(), value.data_type());
835        if let Value::Date(d) = value {
836            assert_eq!("2022-02-21", d.to_string());
837        } else {
838            unreachable!()
839        }
840    }
841
842    #[test]
843    fn test_parse_timestamp_literal() -> Result<()> {
844        match call_parse_string_to_value!(
845            "timestamp_col",
846            "2022-02-22T00:01:01+08:00".to_string(),
847            ConcreteDataType::timestamp_millisecond_datatype()
848        )? {
849            Value::Timestamp(ts) => {
850                assert_eq!(1645459261000, ts.value());
851                assert_eq!(TimeUnit::Millisecond, ts.unit());
852            }
853            _ => {
854                unreachable!()
855            }
856        }
857
858        match call_parse_string_to_value!(
859            "timestamp_col",
860            "2022-02-22T00:01:01+08:00".to_string(),
861            ConcreteDataType::timestamp_datatype(TimeUnit::Second)
862        )? {
863            Value::Timestamp(ts) => {
864                assert_eq!(1645459261, ts.value());
865                assert_eq!(TimeUnit::Second, ts.unit());
866            }
867            _ => {
868                unreachable!()
869            }
870        }
871
872        match call_parse_string_to_value!(
873            "timestamp_col",
874            "2022-02-22T00:01:01+08:00".to_string(),
875            ConcreteDataType::timestamp_datatype(TimeUnit::Microsecond)
876        )? {
877            Value::Timestamp(ts) => {
878                assert_eq!(1645459261000000, ts.value());
879                assert_eq!(TimeUnit::Microsecond, ts.unit());
880            }
881            _ => {
882                unreachable!()
883            }
884        }
885
886        match call_parse_string_to_value!(
887            "timestamp_col",
888            "2022-02-22T00:01:01+08:00".to_string(),
889            ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond)
890        )? {
891            Value::Timestamp(ts) => {
892                assert_eq!(1645459261000000000, ts.value());
893                assert_eq!(TimeUnit::Nanosecond, ts.unit());
894            }
895            _ => {
896                unreachable!()
897            }
898        }
899
900        assert!(
901            call_parse_string_to_value!(
902                "timestamp_col",
903                "2022-02-22T00:01:01+08".to_string(),
904                ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond)
905            )
906            .is_err()
907        );
908
909        // with timezone
910        match call_parse_string_to_value!(
911            "timestamp_col",
912            "2022-02-22T00:01:01".to_string(),
913            ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond),
914            timezone = &Timezone::from_tz_string("Asia/Shanghai").unwrap()
915        )? {
916            Value::Timestamp(ts) => {
917                assert_eq!(1645459261000000000, ts.value());
918                assert_eq!("2022-02-21 16:01:01+0000", ts.to_iso8601_string());
919                assert_eq!(TimeUnit::Nanosecond, ts.unit());
920            }
921            _ => {
922                unreachable!()
923            }
924        }
925        Ok(())
926    }
927
928    #[test]
929    fn test_parse_placeholder_value() {
930        assert!(
931            call_sql_value_to_value!(
932                "test",
933                ConcreteDataType::string_datatype(),
934                &SqlValue::Placeholder("default".into())
935            )
936            .is_err()
937        );
938        assert!(
939            call_sql_value_to_value!(
940                "test",
941                ConcreteDataType::string_datatype(),
942                &SqlValue::Placeholder("default".into()),
943                unary_op = UnaryOperator::Minus
944            )
945            .is_err()
946        );
947        assert!(
948            call_sql_value_to_value!(
949                "test",
950                ConcreteDataType::uint16_datatype(),
951                &SqlValue::Number("3".into(), false),
952                unary_op = UnaryOperator::Minus
953            )
954            .is_err()
955        );
956        assert!(
957            call_sql_value_to_value!(
958                "test",
959                ConcreteDataType::uint16_datatype(),
960                &SqlValue::Number("3".into(), false)
961            )
962            .is_ok()
963        );
964    }
965
966    #[test]
967    fn test_negative_number_at_type_minimum() {
968        for (data_type, digits, expected) in [
969            (
970                ConcreteDataType::int8_datatype(),
971                "128",
972                Value::Int8(i8::MIN),
973            ),
974            (
975                ConcreteDataType::int32_datatype(),
976                "2147483648",
977                Value::Int32(i32::MIN),
978            ),
979            (
980                ConcreteDataType::int64_datatype(),
981                "9223372036854775808",
982                Value::Int64(i64::MIN),
983            ),
984        ] {
985            let value = call_sql_value_to_value!(
986                "test",
987                data_type,
988                &SqlValue::Number(digits.into(), false),
989                unary_op = UnaryOperator::Minus
990            )
991            .unwrap();
992            assert_eq!(value, expected);
993        }
994        let value = call_sql_value_to_value!(
995            "test",
996            ConcreteDataType::uint64_datatype(),
997            &SqlValue::Number("0".into(), false),
998            unary_op = UnaryOperator::Minus
999        )
1000        .unwrap();
1001        assert_eq!(value, Value::UInt64(0));
1002
1003        // 28 characters without the sign, 29 with it: the sign must not move the
1004        // literal to the other decimal parser.
1005        let value = call_sql_value_to_value!(
1006            "test",
1007            ConcreteDataType::decimal128_datatype(38, 30),
1008            &SqlValue::Number("0.01234567890123456789012345".into(), false),
1009            unary_op = UnaryOperator::Minus
1010        )
1011        .unwrap();
1012        assert_eq!(value.to_string(), "-0.012345678901234567890123450000");
1013    }
1014
1015    #[test]
1016    fn test_auto_string_to_numeric() {
1017        // Test with auto_string_to_numeric=true
1018        let sql_val = SqlValue::SingleQuotedString("123".to_string());
1019        let v = call_sql_value_to_value!(
1020            "a",
1021            ConcreteDataType::int32_datatype(),
1022            &sql_val,
1023            auto_string_to_numeric
1024        )
1025        .unwrap();
1026        assert_eq!(Value::Int32(123), v);
1027
1028        // Test with a float string
1029        let sql_val = SqlValue::SingleQuotedString("3.5".to_string());
1030        let v = call_sql_value_to_value!(
1031            "a",
1032            ConcreteDataType::float64_datatype(),
1033            &sql_val,
1034            auto_string_to_numeric
1035        )
1036        .unwrap();
1037        assert_eq!(Value::Float64(OrderedFloat(3.5)), v);
1038
1039        // Test with auto_string_to_numeric=false
1040        let sql_val = SqlValue::SingleQuotedString("123".to_string());
1041        let v = call_sql_value_to_value!("a", ConcreteDataType::int32_datatype(), &sql_val);
1042        assert!(v.is_err());
1043
1044        // Test with an invalid numeric string but auto_string_to_numeric=true
1045        // Should return an error now with the new auto_cast_to_numeric behavior
1046        let sql_val = SqlValue::SingleQuotedString("not_a_number".to_string());
1047        let v = call_sql_value_to_value!(
1048            "a",
1049            ConcreteDataType::int32_datatype(),
1050            &sql_val,
1051            auto_string_to_numeric
1052        );
1053        assert!(v.is_err());
1054
1055        // Test with boolean type
1056        let sql_val = SqlValue::SingleQuotedString("true".to_string());
1057        let v = call_sql_value_to_value!(
1058            "a",
1059            ConcreteDataType::boolean_datatype(),
1060            &sql_val,
1061            auto_string_to_numeric
1062        )
1063        .unwrap();
1064        assert_eq!(Value::Boolean(true), v);
1065
1066        // Non-numeric types should still be handled normally
1067        let sql_val = SqlValue::SingleQuotedString("hello".to_string());
1068        let v = call_sql_value_to_value!(
1069            "a",
1070            ConcreteDataType::string_datatype(),
1071            &sql_val,
1072            auto_string_to_numeric
1073        );
1074        assert!(v.is_ok());
1075    }
1076
1077    #[test]
1078    fn test_sql_number_to_value_timestamp_strict_typing() {
1079        // Test that values are interpreted according to the target column type
1080        let timestamp_type = TimestampType::Millisecond(datatypes::types::TimestampMillisecondType);
1081        let data_type = ConcreteDataType::Timestamp(timestamp_type);
1082
1083        // Valid millisecond timestamp
1084        let millisecond_str = "1747814093865";
1085        let result = sql_number_to_value(&data_type, millisecond_str).unwrap();
1086        if let Value::Timestamp(ts) = result {
1087            assert_eq!(ts.unit(), TimeUnit::Millisecond);
1088            assert_eq!(ts.value(), 1747814093865);
1089        } else {
1090            panic!("Expected timestamp value");
1091        }
1092
1093        // Large value that would overflow when treated as milliseconds should be rejected
1094        let nanosecond_str = "1747814093865000000"; // This is too large for millisecond precision
1095        let result = sql_number_to_value(&data_type, nanosecond_str);
1096        assert!(
1097            result.is_err(),
1098            "Should reject overly large timestamp values"
1099        );
1100    }
1101
1102    #[test]
1103    fn test_sql_number_to_value_timestamp_different_units() {
1104        // Test second precision
1105        let second_type = TimestampType::Second(datatypes::types::TimestampSecondType);
1106        let second_data_type = ConcreteDataType::Timestamp(second_type);
1107
1108        let second_str = "1747814093";
1109        let result = sql_number_to_value(&second_data_type, second_str).unwrap();
1110        if let Value::Timestamp(ts) = result {
1111            assert_eq!(ts.unit(), TimeUnit::Second);
1112            assert_eq!(ts.value(), 1747814093);
1113        } else {
1114            panic!("Expected timestamp value");
1115        }
1116
1117        // Test nanosecond precision
1118        let nanosecond_type = TimestampType::Nanosecond(datatypes::types::TimestampNanosecondType);
1119        let nanosecond_data_type = ConcreteDataType::Timestamp(nanosecond_type);
1120
1121        let nanosecond_str = "1747814093865000000";
1122        let result = sql_number_to_value(&nanosecond_data_type, nanosecond_str).unwrap();
1123        if let Value::Timestamp(ts) = result {
1124            assert_eq!(ts.unit(), TimeUnit::Nanosecond);
1125            assert_eq!(ts.value(), 1747814093865000000);
1126        } else {
1127            panic!("Expected timestamp value");
1128        }
1129    }
1130
1131    #[test]
1132    fn test_timestamp_range_validation() {
1133        // Test that our range checking works correctly
1134        let nanosecond_value = 1747814093865000000i64; // This should be too large for millisecond
1135
1136        // This should work for nanosecond precision
1137        let nanosecond_type = TimestampType::Nanosecond(datatypes::types::TimestampNanosecondType);
1138        let nanosecond_data_type = ConcreteDataType::Timestamp(nanosecond_type);
1139        let result = sql_number_to_value(&nanosecond_data_type, "1747814093865000000");
1140        assert!(
1141            result.is_ok(),
1142            "Nanosecond value should be valid for nanosecond column"
1143        );
1144
1145        // This should fail for millisecond precision (value too large)
1146        let millisecond_type =
1147            TimestampType::Millisecond(datatypes::types::TimestampMillisecondType);
1148        let millisecond_data_type = ConcreteDataType::Timestamp(millisecond_type);
1149        let result = sql_number_to_value(&millisecond_data_type, "1747814093865000000");
1150        assert!(
1151            result.is_err(),
1152            "Nanosecond value should be rejected for millisecond column"
1153        );
1154
1155        // Verify the ranges work as expected
1156        assert!(
1157            nanosecond_value > Timestamp::MAX_MILLISECOND.value(),
1158            "Test value should exceed millisecond range"
1159        );
1160    }
1161}
common_sql/convert.rs

common_sql/
convert.rs
