common_sql/
convert.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::str::FromStr;
16
17use common_time::Timestamp;
18use common_time::timezone::Timezone;
19use datatypes::json::JsonStructureSettings;
20use datatypes::prelude::ConcreteDataType;
21use datatypes::schema::ColumnDefaultConstraint;
22use datatypes::types::{JsonFormat, parse_string_to_jsonb, parse_string_to_vector_type_value};
23use datatypes::value::{OrderedF32, OrderedF64, Value};
24use snafu::{OptionExt, ResultExt, ensure};
25pub use sqlparser::ast::{
26    BinaryOperator, ColumnDef, ColumnOption, ColumnOptionDef, DataType, Expr, Function,
27    FunctionArg, FunctionArgExpr, FunctionArguments, Ident, ObjectName, SqlOption, TableConstraint,
28    TimezoneInfo, UnaryOperator, Value as SqlValue, Visit, VisitMut, Visitor, VisitorMut,
29    visit_expressions_mut, visit_statements_mut,
30};
31
32use crate::error::{
33    ColumnTypeMismatchSnafu, ConvertSqlValueSnafu, ConvertStrSnafu, DatatypeSnafu,
34    DeserializeSnafu, InvalidCastSnafu, InvalidSqlValueSnafu, InvalidUnaryOpSnafu,
35    ParseSqlValueSnafu, Result, TimestampOverflowSnafu, UnsupportedUnaryOpSnafu,
36};
37
38fn parse_sql_number<R: FromStr + std::fmt::Debug>(n: &str) -> Result<R>
39where
40    <R as FromStr>::Err: std::fmt::Debug,
41{
42    match n.parse::<R>() {
43        Ok(n) => Ok(n),
44        Err(e) => ParseSqlValueSnafu {
45            msg: format!("Fail to parse number {n}, {e:?}"),
46        }
47        .fail(),
48    }
49}
50
51macro_rules! parse_number_to_value {
52    ($data_type: expr, $n: ident,  $(($Type: ident, $PrimitiveType: ident, $Target: ident)), +) => {
53        match $data_type {
54            $(
55                ConcreteDataType::$Type(_) => {
56                    let n  = parse_sql_number::<$PrimitiveType>($n)?;
57                    Ok(Value::$Type($Target::from(n)))
58                },
59            )+
60            ConcreteDataType::Timestamp(t) => {
61                let n = parse_sql_number::<i64>($n)?;
62                let timestamp = Timestamp::new(n, t.unit());
63
64                // Check if the value is within the valid range for the target unit
65                if Timestamp::is_overflow(n, t.unit()) {
66                    return TimestampOverflowSnafu {
67                        timestamp,
68                        target_unit: t.unit(),
69                    }.fail();
70                }
71
72                Ok(Value::Timestamp(timestamp))
73            },
74            // TODO(QuenKar): This could need to be optimized
75            // if this from_str function is slow,
76            // we can implement parse decimal string with precision and scale manually.
77            ConcreteDataType::Decimal128(_) => {
78                if let Ok(val) = common_decimal::Decimal128::from_str($n) {
79                    Ok(Value::Decimal128(val))
80                } else {
81                    ParseSqlValueSnafu {
82                        msg: format!("Fail to parse number {}, invalid column type: {:?}",
83                                        $n, $data_type)
84                    }.fail()
85                }
86            }
87            // It's valid for MySQL JDBC to send "0" and "1" for boolean types, so adapt to that.
88            ConcreteDataType::Boolean(_) => {
89                match $n {
90                    "0" => Ok(Value::Boolean(false)),
91                    "1" => Ok(Value::Boolean(true)),
92                    _ => ParseSqlValueSnafu {
93                        msg: format!("Failed to parse number '{}' to boolean column type", $n)}.fail(),
94                }
95            }
96            _ => ParseSqlValueSnafu {
97                msg: format!("Fail to parse number {}, invalid column type: {:?}",
98                                $n, $data_type
99                )}.fail(),
100        }
101    }
102}
103
104/// Convert a sql value into datatype's value
105pub(crate) fn sql_number_to_value(data_type: &ConcreteDataType, n: &str) -> Result<Value> {
106    parse_number_to_value!(
107        data_type,
108        n,
109        (UInt8, u8, u8),
110        (UInt16, u16, u16),
111        (UInt32, u32, u32),
112        (UInt64, u64, u64),
113        (Int8, i8, i8),
114        (Int16, i16, i16),
115        (Int32, i32, i32),
116        (Int64, i64, i64),
117        (Float64, f64, OrderedF64),
118        (Float32, f32, OrderedF32)
119    )
120    // TODO(hl): also Date/DateTime
121}
122
123/// Converts SQL value to value according to the data type.
124/// If `auto_string_to_numeric` is true, tries to cast the string value to numeric values,
125/// and returns error if the cast fails.
126pub fn sql_value_to_value(
127    column_name: &str,
128    data_type: &ConcreteDataType,
129    sql_val: &SqlValue,
130    timezone: Option<&Timezone>,
131    unary_op: Option<UnaryOperator>,
132    auto_string_to_numeric: bool,
133) -> Result<Value> {
134    let mut value = match sql_val {
135        SqlValue::Number(n, _) => sql_number_to_value(data_type, n)?,
136        SqlValue::Null => Value::Null,
137        SqlValue::Boolean(b) => {
138            ensure!(
139                data_type.is_boolean(),
140                ColumnTypeMismatchSnafu {
141                    column_name,
142                    expect: data_type.clone(),
143                    actual: ConcreteDataType::boolean_datatype(),
144                }
145            );
146
147            (*b).into()
148        }
149        SqlValue::DoubleQuotedString(s) | SqlValue::SingleQuotedString(s) => parse_string_to_value(
150            column_name,
151            s.clone(),
152            data_type,
153            timezone,
154            auto_string_to_numeric,
155        )?,
156        SqlValue::HexStringLiteral(s) => {
157            // Should not directly write binary into json column
158            ensure!(
159                !matches!(data_type, ConcreteDataType::Json(_)),
160                ColumnTypeMismatchSnafu {
161                    column_name,
162                    expect: ConcreteDataType::binary_datatype(),
163                    actual: ConcreteDataType::json_datatype(),
164                }
165            );
166
167            parse_hex_string(s)?
168        }
169        SqlValue::Placeholder(s) => return InvalidSqlValueSnafu { value: s }.fail(),
170
171        // TODO(dennis): supports binary string
172        _ => {
173            return ConvertSqlValueSnafu {
174                value: sql_val.clone(),
175                datatype: data_type.clone(),
176            }
177            .fail();
178        }
179    };
180
181    if let Some(unary_op) = unary_op {
182        match unary_op {
183            UnaryOperator::Plus | UnaryOperator::Minus | UnaryOperator::Not => {}
184            _ => {
185                return UnsupportedUnaryOpSnafu { unary_op }.fail();
186            }
187        }
188
189        match value {
190            Value::Null => {}
191            Value::Boolean(bool) => match unary_op {
192                UnaryOperator::Not => value = Value::Boolean(!bool),
193                _ => {
194                    return InvalidUnaryOpSnafu { unary_op, value }.fail();
195                }
196            },
197            Value::UInt8(_)
198            | Value::UInt16(_)
199            | Value::UInt32(_)
200            | Value::UInt64(_)
201            | Value::Int8(_)
202            | Value::Int16(_)
203            | Value::Int32(_)
204            | Value::Int64(_)
205            | Value::Float32(_)
206            | Value::Float64(_)
207            | Value::Decimal128(_)
208            | Value::Date(_)
209            | Value::Timestamp(_)
210            | Value::Time(_)
211            | Value::Duration(_)
212            | Value::IntervalYearMonth(_)
213            | Value::IntervalDayTime(_)
214            | Value::IntervalMonthDayNano(_) => match unary_op {
215                UnaryOperator::Plus => {}
216                UnaryOperator::Minus => {
217                    value = value
218                        .try_negative()
219                        .with_context(|| InvalidUnaryOpSnafu { unary_op, value })?;
220                }
221                _ => return InvalidUnaryOpSnafu { unary_op, value }.fail(),
222            },
223
224            Value::String(_)
225            | Value::Binary(_)
226            | Value::List(_)
227            | Value::Struct(_)
228            | Value::Json(_) => {
229                return InvalidUnaryOpSnafu { unary_op, value }.fail();
230            }
231        }
232    }
233
234    let value_datatype = value.data_type();
235    // The datatype of json value is determined by its actual data, so we can't simply "cast" it here.
236    if value_datatype.is_json() || value_datatype == *data_type {
237        Ok(value)
238    } else {
239        datatypes::types::cast(value, data_type).with_context(|_| InvalidCastSnafu {
240            sql_value: sql_val.clone(),
241            datatype: data_type,
242        })
243    }
244}
245
246pub(crate) fn parse_string_to_value(
247    column_name: &str,
248    s: String,
249    data_type: &ConcreteDataType,
250    timezone: Option<&Timezone>,
251    auto_string_to_numeric: bool,
252) -> Result<Value> {
253    if auto_string_to_numeric && let Some(value) = auto_cast_to_numeric(&s, data_type)? {
254        return Ok(value);
255    }
256
257    ensure!(
258        data_type.is_stringifiable(),
259        ColumnTypeMismatchSnafu {
260            column_name,
261            expect: data_type.clone(),
262            actual: ConcreteDataType::string_datatype(),
263        }
264    );
265
266    match data_type {
267        ConcreteDataType::String(_) => Ok(Value::String(s.into())),
268        ConcreteDataType::Date(_) => {
269            if let Ok(date) = common_time::date::Date::from_str(&s, timezone) {
270                Ok(Value::Date(date))
271            } else {
272                ParseSqlValueSnafu {
273                    msg: format!("Failed to parse {s} to Date value"),
274                }
275                .fail()
276            }
277        }
278        ConcreteDataType::Timestamp(t) => {
279            if let Ok(ts) = Timestamp::from_str(&s, timezone) {
280                Ok(Value::Timestamp(ts.convert_to(t.unit()).context(
281                    TimestampOverflowSnafu {
282                        timestamp: ts,
283                        target_unit: t.unit(),
284                    },
285                )?))
286            } else if let Ok(ts) = i64::from_str(s.as_str()) {
287                Ok(Value::Timestamp(Timestamp::new(ts, t.unit())))
288            } else {
289                ParseSqlValueSnafu {
290                    msg: format!("Failed to parse {s} to Timestamp value"),
291                }
292                .fail()
293            }
294        }
295        ConcreteDataType::Decimal128(_) => {
296            if let Ok(val) = common_decimal::Decimal128::from_str(&s) {
297                Ok(Value::Decimal128(val))
298            } else {
299                ParseSqlValueSnafu {
300                    msg: format!("Fail to parse number {s} to Decimal128 value"),
301                }
302                .fail()
303            }
304        }
305        ConcreteDataType::Binary(_) => Ok(Value::Binary(s.as_bytes().into())),
306        ConcreteDataType::Json(j) => {
307            match &j.format {
308                JsonFormat::Jsonb => {
309                    let v = parse_string_to_jsonb(&s).context(DatatypeSnafu)?;
310                    Ok(Value::Binary(v.into()))
311                }
312                JsonFormat::Native(_inner) => {
313                    // Always use the structured version at this level.
314                    let serde_json_value =
315                        serde_json::from_str(&s).context(DeserializeSnafu { json: s })?;
316                    let json_structure_settings = JsonStructureSettings::Structured(None);
317                    json_structure_settings
318                        .encode(serde_json_value)
319                        .context(DatatypeSnafu)
320                }
321            }
322        }
323        ConcreteDataType::Vector(d) => {
324            let v = parse_string_to_vector_type_value(&s, Some(d.dim)).context(DatatypeSnafu)?;
325            Ok(Value::Binary(v.into()))
326        }
327        _ => ParseSqlValueSnafu {
328            msg: format!("Failed to parse {s} to {data_type} value"),
329        }
330        .fail(),
331    }
332}
333
334/// Casts string to value of specified numeric data type.
335/// If the string cannot be parsed, returns an error.
336///
337/// Returns None if the data type doesn't support auto casting.
338pub(crate) fn auto_cast_to_numeric(s: &str, data_type: &ConcreteDataType) -> Result<Option<Value>> {
339    let value = match data_type {
340        ConcreteDataType::Boolean(_) => s.parse::<bool>().map(Value::Boolean).ok(),
341        ConcreteDataType::Int8(_) => s.parse::<i8>().map(Value::Int8).ok(),
342        ConcreteDataType::Int16(_) => s.parse::<i16>().map(Value::Int16).ok(),
343        ConcreteDataType::Int32(_) => s.parse::<i32>().map(Value::Int32).ok(),
344        ConcreteDataType::Int64(_) => s.parse::<i64>().map(Value::Int64).ok(),
345        ConcreteDataType::UInt8(_) => s.parse::<u8>().map(Value::UInt8).ok(),
346        ConcreteDataType::UInt16(_) => s.parse::<u16>().map(Value::UInt16).ok(),
347        ConcreteDataType::UInt32(_) => s.parse::<u32>().map(Value::UInt32).ok(),
348        ConcreteDataType::UInt64(_) => s.parse::<u64>().map(Value::UInt64).ok(),
349        ConcreteDataType::Float32(_) => s
350            .parse::<f32>()
351            .map(|v| Value::Float32(OrderedF32::from(v)))
352            .ok(),
353        ConcreteDataType::Float64(_) => s
354            .parse::<f64>()
355            .map(|v| Value::Float64(OrderedF64::from(v)))
356            .ok(),
357        _ => return Ok(None),
358    };
359
360    match value {
361        Some(value) => Ok(Some(value)),
362        None => ConvertStrSnafu {
363            value: s,
364            datatype: data_type.clone(),
365        }
366        .fail(),
367    }
368}
369
370pub(crate) fn parse_hex_string(s: &str) -> Result<Value> {
371    match hex::decode(s) {
372        Ok(b) => Ok(Value::Binary(common_base::bytes::Bytes::from(b))),
373        Err(hex::FromHexError::InvalidHexCharacter { c, index }) => ParseSqlValueSnafu {
374            msg: format!(
375                "Fail to parse hex string to Byte: invalid character {c:?} at position {index}"
376            ),
377        }
378        .fail(),
379        Err(hex::FromHexError::OddLength) => ParseSqlValueSnafu {
380            msg: "Fail to parse hex string to Byte: odd number of digits".to_string(),
381        }
382        .fail(),
383        Err(e) => ParseSqlValueSnafu {
384            msg: format!("Fail to parse hex string to Byte {s}, {e:?}"),
385        }
386        .fail(),
387    }
388}
389
390/// Deserialize default constraint from json bytes
391pub fn deserialize_default_constraint(
392    bytes: &[u8],
393    column_name: &str,
394    data_type: &ConcreteDataType,
395) -> Result<Option<ColumnDefaultConstraint>> {
396    let json = String::from_utf8_lossy(bytes);
397    let default_constraint = serde_json::from_str(&json).context(DeserializeSnafu { json })?;
398    let column_def = sqlparser::ast::ColumnOptionDef {
399        name: None,
400        option: sqlparser::ast::ColumnOption::Default(default_constraint),
401    };
402
403    crate::default_constraint::parse_column_default_constraint(
404        column_name,
405        data_type,
406        &[column_def],
407        None,
408    )
409}
410
411#[cfg(test)]
412mod test {
413    use common_base::bytes::Bytes;
414    use common_time::timestamp::TimeUnit;
415    use datatypes::types::TimestampType;
416    use datatypes::value::OrderedFloat;
417
418    use super::*;
419
420    #[test]
421    fn test_string_to_value_auto_numeric() {
422        // Test string to boolean with auto cast
423        let result = parse_string_to_value(
424            "col",
425            "true".to_string(),
426            &ConcreteDataType::boolean_datatype(),
427            None,
428            true,
429        )
430        .unwrap();
431        assert_eq!(Value::Boolean(true), result);
432
433        // Test invalid string to boolean with auto cast
434        let result = parse_string_to_value(
435            "col",
436            "not_a_boolean".to_string(),
437            &ConcreteDataType::boolean_datatype(),
438            None,
439            true,
440        );
441        assert!(result.is_err());
442
443        // Test string to int8
444        let result = parse_string_to_value(
445            "col",
446            "42".to_string(),
447            &ConcreteDataType::int8_datatype(),
448            None,
449            true,
450        )
451        .unwrap();
452        assert_eq!(Value::Int8(42), result);
453
454        // Test invalid string to int8 with auto cast
455        let result = parse_string_to_value(
456            "col",
457            "not_an_int8".to_string(),
458            &ConcreteDataType::int8_datatype(),
459            None,
460            true,
461        );
462        assert!(result.is_err());
463
464        // Test string to int16
465        let result = parse_string_to_value(
466            "col",
467            "1000".to_string(),
468            &ConcreteDataType::int16_datatype(),
469            None,
470            true,
471        )
472        .unwrap();
473        assert_eq!(Value::Int16(1000), result);
474
475        // Test invalid string to int16 with auto cast
476        let result = parse_string_to_value(
477            "col",
478            "not_an_int16".to_string(),
479            &ConcreteDataType::int16_datatype(),
480            None,
481            true,
482        );
483        assert!(result.is_err());
484
485        // Test string to int32
486        let result = parse_string_to_value(
487            "col",
488            "100000".to_string(),
489            &ConcreteDataType::int32_datatype(),
490            None,
491            true,
492        )
493        .unwrap();
494        assert_eq!(Value::Int32(100000), result);
495
496        // Test invalid string to int32 with auto cast
497        let result = parse_string_to_value(
498            "col",
499            "not_an_int32".to_string(),
500            &ConcreteDataType::int32_datatype(),
501            None,
502            true,
503        );
504        assert!(result.is_err());
505
506        // Test string to int64
507        let result = parse_string_to_value(
508            "col",
509            "1000000".to_string(),
510            &ConcreteDataType::int64_datatype(),
511            None,
512            true,
513        )
514        .unwrap();
515        assert_eq!(Value::Int64(1000000), result);
516
517        // Test invalid string to int64 with auto cast
518        let result = parse_string_to_value(
519            "col",
520            "not_an_int64".to_string(),
521            &ConcreteDataType::int64_datatype(),
522            None,
523            true,
524        );
525        assert!(result.is_err());
526
527        // Test string to uint8
528        let result = parse_string_to_value(
529            "col",
530            "200".to_string(),
531            &ConcreteDataType::uint8_datatype(),
532            None,
533            true,
534        )
535        .unwrap();
536        assert_eq!(Value::UInt8(200), result);
537
538        // Test invalid string to uint8 with auto cast
539        let result = parse_string_to_value(
540            "col",
541            "not_a_uint8".to_string(),
542            &ConcreteDataType::uint8_datatype(),
543            None,
544            true,
545        );
546        assert!(result.is_err());
547
548        // Test string to uint16
549        let result = parse_string_to_value(
550            "col",
551            "60000".to_string(),
552            &ConcreteDataType::uint16_datatype(),
553            None,
554            true,
555        )
556        .unwrap();
557        assert_eq!(Value::UInt16(60000), result);
558
559        // Test invalid string to uint16 with auto cast
560        let result = parse_string_to_value(
561            "col",
562            "not_a_uint16".to_string(),
563            &ConcreteDataType::uint16_datatype(),
564            None,
565            true,
566        );
567        assert!(result.is_err());
568
569        // Test string to uint32
570        let result = parse_string_to_value(
571            "col",
572            "4000000000".to_string(),
573            &ConcreteDataType::uint32_datatype(),
574            None,
575            true,
576        )
577        .unwrap();
578        assert_eq!(Value::UInt32(4000000000), result);
579
580        // Test invalid string to uint32 with auto cast
581        let result = parse_string_to_value(
582            "col",
583            "not_a_uint32".to_string(),
584            &ConcreteDataType::uint32_datatype(),
585            None,
586            true,
587        );
588        assert!(result.is_err());
589
590        // Test string to uint64
591        let result = parse_string_to_value(
592            "col",
593            "18446744073709551615".to_string(),
594            &ConcreteDataType::uint64_datatype(),
595            None,
596            true,
597        )
598        .unwrap();
599        assert_eq!(Value::UInt64(18446744073709551615), result);
600
601        // Test invalid string to uint64 with auto cast
602        let result = parse_string_to_value(
603            "col",
604            "not_a_uint64".to_string(),
605            &ConcreteDataType::uint64_datatype(),
606            None,
607            true,
608        );
609        assert!(result.is_err());
610
611        // Test string to float32
612        let result = parse_string_to_value(
613            "col",
614            "3.5".to_string(),
615            &ConcreteDataType::float32_datatype(),
616            None,
617            true,
618        )
619        .unwrap();
620        assert_eq!(Value::Float32(OrderedF32::from(3.5)), result);
621
622        // Test invalid string to float32 with auto cast
623        let result = parse_string_to_value(
624            "col",
625            "not_a_float32".to_string(),
626            &ConcreteDataType::float32_datatype(),
627            None,
628            true,
629        );
630        assert!(result.is_err());
631
632        // Test string to float64
633        let result = parse_string_to_value(
634            "col",
635            "3.5".to_string(),
636            &ConcreteDataType::float64_datatype(),
637            None,
638            true,
639        )
640        .unwrap();
641        assert_eq!(Value::Float64(OrderedF64::from(3.5)), result);
642
643        // Test invalid string to float64 with auto cast
644        let result = parse_string_to_value(
645            "col",
646            "not_a_float64".to_string(),
647            &ConcreteDataType::float64_datatype(),
648            None,
649            true,
650        );
651        assert!(result.is_err());
652    }
653
654    #[test]
655    fn test_sql_value_to_value() {
656        let sql_val = SqlValue::Null;
657        assert_eq!(
658            Value::Null,
659            sql_value_to_value(
660                "a",
661                &ConcreteDataType::float64_datatype(),
662                &sql_val,
663                None,
664                None,
665                false
666            )
667            .unwrap()
668        );
669
670        let sql_val = SqlValue::Boolean(true);
671        assert_eq!(
672            Value::Boolean(true),
673            sql_value_to_value(
674                "a",
675                &ConcreteDataType::boolean_datatype(),
676                &sql_val,
677                None,
678                None,
679                false
680            )
681            .unwrap()
682        );
683
684        let sql_val = SqlValue::Number("3.0".to_string(), false);
685        assert_eq!(
686            Value::Float64(OrderedFloat(3.0)),
687            sql_value_to_value(
688                "a",
689                &ConcreteDataType::float64_datatype(),
690                &sql_val,
691                None,
692                None,
693                false
694            )
695            .unwrap()
696        );
697
698        let sql_val = SqlValue::Number("3.0".to_string(), false);
699        let v = sql_value_to_value(
700            "a",
701            &ConcreteDataType::boolean_datatype(),
702            &sql_val,
703            None,
704            None,
705            false,
706        );
707        assert!(v.is_err());
708        assert!(format!("{v:?}").contains("Failed to parse number '3.0' to boolean column type"));
709
710        let sql_val = SqlValue::Boolean(true);
711        let v = sql_value_to_value(
712            "a",
713            &ConcreteDataType::float64_datatype(),
714            &sql_val,
715            None,
716            None,
717            false,
718        );
719        assert!(v.is_err());
720        assert!(
721            format!("{v:?}").contains(
722                "Column a expect type: Float64(Float64Type), actual: Boolean(BooleanType)"
723            ),
724            "v is {v:?}",
725        );
726
727        let sql_val = SqlValue::HexStringLiteral("48656c6c6f20776f726c6421".to_string());
728        let v = sql_value_to_value(
729            "a",
730            &ConcreteDataType::binary_datatype(),
731            &sql_val,
732            None,
733            None,
734            false,
735        )
736        .unwrap();
737        assert_eq!(Value::Binary(Bytes::from(b"Hello world!".as_slice())), v);
738
739        let sql_val = SqlValue::DoubleQuotedString("MorningMyFriends".to_string());
740        let v = sql_value_to_value(
741            "a",
742            &ConcreteDataType::binary_datatype(),
743            &sql_val,
744            None,
745            None,
746            false,
747        )
748        .unwrap();
749        assert_eq!(
750            Value::Binary(Bytes::from(b"MorningMyFriends".as_slice())),
751            v
752        );
753
754        let sql_val = SqlValue::HexStringLiteral("9AF".to_string());
755        let v = sql_value_to_value(
756            "a",
757            &ConcreteDataType::binary_datatype(),
758            &sql_val,
759            None,
760            None,
761            false,
762        );
763        assert!(v.is_err());
764        assert!(
765            format!("{v:?}").contains("odd number of digits"),
766            "v is {v:?}"
767        );
768
769        let sql_val = SqlValue::HexStringLiteral("AG".to_string());
770        let v = sql_value_to_value(
771            "a",
772            &ConcreteDataType::binary_datatype(),
773            &sql_val,
774            None,
775            None,
776            false,
777        );
778        assert!(v.is_err());
779        assert!(format!("{v:?}").contains("invalid character"), "v is {v:?}",);
780
781        let sql_val = SqlValue::DoubleQuotedString("MorningMyFriends".to_string());
782        let v = sql_value_to_value(
783            "a",
784            &ConcreteDataType::json_datatype(),
785            &sql_val,
786            None,
787            None,
788            false,
789        );
790        assert!(v.is_err());
791
792        let sql_val = SqlValue::DoubleQuotedString(r#"{"a":"b"}"#.to_string());
793        let v = sql_value_to_value(
794            "a",
795            &ConcreteDataType::json_datatype(),
796            &sql_val,
797            None,
798            None,
799            false,
800        )
801        .unwrap();
802        assert_eq!(
803            Value::Binary(Bytes::from(
804                jsonb::parse_value(r#"{"a":"b"}"#.as_bytes())
805                    .unwrap()
806                    .to_vec()
807                    .as_slice()
808            )),
809            v
810        );
811    }
812
813    #[test]
814    fn test_parse_json_to_jsonb() {
815        match parse_string_to_value(
816            "json_col",
817            r#"{"a": "b"}"#.to_string(),
818            &ConcreteDataType::json_datatype(),
819            None,
820            false,
821        ) {
822            Ok(Value::Binary(b)) => {
823                assert_eq!(
824                    b,
825                    jsonb::parse_value(r#"{"a": "b"}"#.as_bytes())
826                        .unwrap()
827                        .to_vec()
828                );
829            }
830            _ => {
831                unreachable!()
832            }
833        }
834
835        assert!(
836            parse_string_to_value(
837                "json_col",
838                r#"Nicola Kovac is the best rifler in the world"#.to_string(),
839                &ConcreteDataType::json_datatype(),
840                None,
841                false,
842            )
843            .is_err()
844        )
845    }
846
847    #[test]
848    fn test_sql_number_to_value() {
849        let v = sql_number_to_value(&ConcreteDataType::float64_datatype(), "3.0").unwrap();
850        assert_eq!(Value::Float64(OrderedFloat(3.0)), v);
851
852        let v = sql_number_to_value(&ConcreteDataType::int32_datatype(), "999").unwrap();
853        assert_eq!(Value::Int32(999), v);
854
855        let v = sql_number_to_value(
856            &ConcreteDataType::timestamp_nanosecond_datatype(),
857            "1073741821",
858        )
859        .unwrap();
860        assert_eq!(Value::Timestamp(Timestamp::new_nanosecond(1073741821)), v);
861
862        let v = sql_number_to_value(
863            &ConcreteDataType::timestamp_millisecond_datatype(),
864            "999999",
865        )
866        .unwrap();
867        assert_eq!(Value::Timestamp(Timestamp::new_millisecond(999999)), v);
868
869        let v = sql_number_to_value(&ConcreteDataType::string_datatype(), "999");
870        assert!(v.is_err(), "parse value error is: {v:?}");
871
872        let v = sql_number_to_value(&ConcreteDataType::boolean_datatype(), "0").unwrap();
873        assert_eq!(v, Value::Boolean(false));
874        let v = sql_number_to_value(&ConcreteDataType::boolean_datatype(), "1").unwrap();
875        assert_eq!(v, Value::Boolean(true));
876        assert!(sql_number_to_value(&ConcreteDataType::boolean_datatype(), "2").is_err());
877    }
878
879    #[test]
880    fn test_parse_date_literal() {
881        let value = sql_value_to_value(
882            "date",
883            &ConcreteDataType::date_datatype(),
884            &SqlValue::DoubleQuotedString("2022-02-22".to_string()),
885            None,
886            None,
887            false,
888        )
889        .unwrap();
890        assert_eq!(ConcreteDataType::date_datatype(), value.data_type());
891        if let Value::Date(d) = value {
892            assert_eq!("2022-02-22", d.to_string());
893        } else {
894            unreachable!()
895        }
896
897        // with timezone
898        let value = sql_value_to_value(
899            "date",
900            &ConcreteDataType::date_datatype(),
901            &SqlValue::DoubleQuotedString("2022-02-22".to_string()),
902            Some(&Timezone::from_tz_string("+07:00").unwrap()),
903            None,
904            false,
905        )
906        .unwrap();
907        assert_eq!(ConcreteDataType::date_datatype(), value.data_type());
908        if let Value::Date(d) = value {
909            assert_eq!("2022-02-21", d.to_string());
910        } else {
911            unreachable!()
912        }
913    }
914
915    #[test]
916    fn test_parse_timestamp_literal() {
917        match parse_string_to_value(
918            "timestamp_col",
919            "2022-02-22T00:01:01+08:00".to_string(),
920            &ConcreteDataType::timestamp_millisecond_datatype(),
921            None,
922            false,
923        )
924        .unwrap()
925        {
926            Value::Timestamp(ts) => {
927                assert_eq!(1645459261000, ts.value());
928                assert_eq!(TimeUnit::Millisecond, ts.unit());
929            }
930            _ => {
931                unreachable!()
932            }
933        }
934
935        match parse_string_to_value(
936            "timestamp_col",
937            "2022-02-22T00:01:01+08:00".to_string(),
938            &ConcreteDataType::timestamp_datatype(TimeUnit::Second),
939            None,
940            false,
941        )
942        .unwrap()
943        {
944            Value::Timestamp(ts) => {
945                assert_eq!(1645459261, ts.value());
946                assert_eq!(TimeUnit::Second, ts.unit());
947            }
948            _ => {
949                unreachable!()
950            }
951        }
952
953        match parse_string_to_value(
954            "timestamp_col",
955            "2022-02-22T00:01:01+08:00".to_string(),
956            &ConcreteDataType::timestamp_datatype(TimeUnit::Microsecond),
957            None,
958            false,
959        )
960        .unwrap()
961        {
962            Value::Timestamp(ts) => {
963                assert_eq!(1645459261000000, ts.value());
964                assert_eq!(TimeUnit::Microsecond, ts.unit());
965            }
966            _ => {
967                unreachable!()
968            }
969        }
970
971        match parse_string_to_value(
972            "timestamp_col",
973            "2022-02-22T00:01:01+08:00".to_string(),
974            &ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond),
975            None,
976            false,
977        )
978        .unwrap()
979        {
980            Value::Timestamp(ts) => {
981                assert_eq!(1645459261000000000, ts.value());
982                assert_eq!(TimeUnit::Nanosecond, ts.unit());
983            }
984            _ => {
985                unreachable!()
986            }
987        }
988
989        assert!(
990            parse_string_to_value(
991                "timestamp_col",
992                "2022-02-22T00:01:01+08".to_string(),
993                &ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond),
994                None,
995                false,
996            )
997            .is_err()
998        );
999
1000        // with timezone
1001        match parse_string_to_value(
1002            "timestamp_col",
1003            "2022-02-22T00:01:01".to_string(),
1004            &ConcreteDataType::timestamp_datatype(TimeUnit::Nanosecond),
1005            Some(&Timezone::from_tz_string("Asia/Shanghai").unwrap()),
1006            false,
1007        )
1008        .unwrap()
1009        {
1010            Value::Timestamp(ts) => {
1011                assert_eq!(1645459261000000000, ts.value());
1012                assert_eq!("2022-02-21 16:01:01+0000", ts.to_iso8601_string());
1013                assert_eq!(TimeUnit::Nanosecond, ts.unit());
1014            }
1015            _ => {
1016                unreachable!()
1017            }
1018        }
1019    }
1020
1021    #[test]
1022    fn test_parse_placeholder_value() {
1023        assert!(
1024            sql_value_to_value(
1025                "test",
1026                &ConcreteDataType::string_datatype(),
1027                &SqlValue::Placeholder("default".into()),
1028                None,
1029                None,
1030                false
1031            )
1032            .is_err()
1033        );
1034        assert!(
1035            sql_value_to_value(
1036                "test",
1037                &ConcreteDataType::string_datatype(),
1038                &SqlValue::Placeholder("default".into()),
1039                None,
1040                Some(UnaryOperator::Minus),
1041                false
1042            )
1043            .is_err()
1044        );
1045        assert!(
1046            sql_value_to_value(
1047                "test",
1048                &ConcreteDataType::uint16_datatype(),
1049                &SqlValue::Number("3".into(), false),
1050                None,
1051                Some(UnaryOperator::Minus),
1052                false
1053            )
1054            .is_err()
1055        );
1056        assert!(
1057            sql_value_to_value(
1058                "test",
1059                &ConcreteDataType::uint16_datatype(),
1060                &SqlValue::Number("3".into(), false),
1061                None,
1062                None,
1063                false
1064            )
1065            .is_ok()
1066        );
1067    }
1068
1069    #[test]
1070    fn test_auto_string_to_numeric() {
1071        // Test with auto_string_to_numeric=true
1072        let sql_val = SqlValue::SingleQuotedString("123".to_string());
1073        let v = sql_value_to_value(
1074            "a",
1075            &ConcreteDataType::int32_datatype(),
1076            &sql_val,
1077            None,
1078            None,
1079            true,
1080        )
1081        .unwrap();
1082        assert_eq!(Value::Int32(123), v);
1083
1084        // Test with a float string
1085        let sql_val = SqlValue::SingleQuotedString("3.5".to_string());
1086        let v = sql_value_to_value(
1087            "a",
1088            &ConcreteDataType::float64_datatype(),
1089            &sql_val,
1090            None,
1091            None,
1092            true,
1093        )
1094        .unwrap();
1095        assert_eq!(Value::Float64(OrderedFloat(3.5)), v);
1096
1097        // Test with auto_string_to_numeric=false
1098        let sql_val = SqlValue::SingleQuotedString("123".to_string());
1099        let v = sql_value_to_value(
1100            "a",
1101            &ConcreteDataType::int32_datatype(),
1102            &sql_val,
1103            None,
1104            None,
1105            false,
1106        );
1107        assert!(v.is_err());
1108
1109        // Test with an invalid numeric string but auto_string_to_numeric=true
1110        // Should return an error now with the new auto_cast_to_numeric behavior
1111        let sql_val = SqlValue::SingleQuotedString("not_a_number".to_string());
1112        let v = sql_value_to_value(
1113            "a",
1114            &ConcreteDataType::int32_datatype(),
1115            &sql_val,
1116            None,
1117            None,
1118            true,
1119        );
1120        assert!(v.is_err());
1121
1122        // Test with boolean type
1123        let sql_val = SqlValue::SingleQuotedString("true".to_string());
1124        let v = sql_value_to_value(
1125            "a",
1126            &ConcreteDataType::boolean_datatype(),
1127            &sql_val,
1128            None,
1129            None,
1130            true,
1131        )
1132        .unwrap();
1133        assert_eq!(Value::Boolean(true), v);
1134
1135        // Non-numeric types should still be handled normally
1136        let sql_val = SqlValue::SingleQuotedString("hello".to_string());
1137        let v = sql_value_to_value(
1138            "a",
1139            &ConcreteDataType::string_datatype(),
1140            &sql_val,
1141            None,
1142            None,
1143            true,
1144        );
1145        assert!(v.is_ok());
1146    }
1147
1148    #[test]
1149    fn test_sql_number_to_value_timestamp_strict_typing() {
1150        // Test that values are interpreted according to the target column type
1151        let timestamp_type = TimestampType::Millisecond(datatypes::types::TimestampMillisecondType);
1152        let data_type = ConcreteDataType::Timestamp(timestamp_type);
1153
1154        // Valid millisecond timestamp
1155        let millisecond_str = "1747814093865";
1156        let result = sql_number_to_value(&data_type, millisecond_str).unwrap();
1157        if let Value::Timestamp(ts) = result {
1158            assert_eq!(ts.unit(), TimeUnit::Millisecond);
1159            assert_eq!(ts.value(), 1747814093865);
1160        } else {
1161            panic!("Expected timestamp value");
1162        }
1163
1164        // Large value that would overflow when treated as milliseconds should be rejected
1165        let nanosecond_str = "1747814093865000000"; // This is too large for millisecond precision
1166        let result = sql_number_to_value(&data_type, nanosecond_str);
1167        assert!(
1168            result.is_err(),
1169            "Should reject overly large timestamp values"
1170        );
1171    }
1172
1173    #[test]
1174    fn test_sql_number_to_value_timestamp_different_units() {
1175        // Test second precision
1176        let second_type = TimestampType::Second(datatypes::types::TimestampSecondType);
1177        let second_data_type = ConcreteDataType::Timestamp(second_type);
1178
1179        let second_str = "1747814093";
1180        let result = sql_number_to_value(&second_data_type, second_str).unwrap();
1181        if let Value::Timestamp(ts) = result {
1182            assert_eq!(ts.unit(), TimeUnit::Second);
1183            assert_eq!(ts.value(), 1747814093);
1184        } else {
1185            panic!("Expected timestamp value");
1186        }
1187
1188        // Test nanosecond precision
1189        let nanosecond_type = TimestampType::Nanosecond(datatypes::types::TimestampNanosecondType);
1190        let nanosecond_data_type = ConcreteDataType::Timestamp(nanosecond_type);
1191
1192        let nanosecond_str = "1747814093865000000";
1193        let result = sql_number_to_value(&nanosecond_data_type, nanosecond_str).unwrap();
1194        if let Value::Timestamp(ts) = result {
1195            assert_eq!(ts.unit(), TimeUnit::Nanosecond);
1196            assert_eq!(ts.value(), 1747814093865000000);
1197        } else {
1198            panic!("Expected timestamp value");
1199        }
1200    }
1201
1202    #[test]
1203    fn test_timestamp_range_validation() {
1204        // Test that our range checking works correctly
1205        let nanosecond_value = 1747814093865000000i64; // This should be too large for millisecond
1206
1207        // This should work for nanosecond precision
1208        let nanosecond_type = TimestampType::Nanosecond(datatypes::types::TimestampNanosecondType);
1209        let nanosecond_data_type = ConcreteDataType::Timestamp(nanosecond_type);
1210        let result = sql_number_to_value(&nanosecond_data_type, "1747814093865000000");
1211        assert!(
1212            result.is_ok(),
1213            "Nanosecond value should be valid for nanosecond column"
1214        );
1215
1216        // This should fail for millisecond precision (value too large)
1217        let millisecond_type =
1218            TimestampType::Millisecond(datatypes::types::TimestampMillisecondType);
1219        let millisecond_data_type = ConcreteDataType::Timestamp(millisecond_type);
1220        let result = sql_number_to_value(&millisecond_data_type, "1747814093865000000");
1221        assert!(
1222            result.is_err(),
1223            "Nanosecond value should be rejected for millisecond column"
1224        );
1225
1226        // Verify the ranges work as expected
1227        assert!(
1228            nanosecond_value > Timestamp::MAX_MILLISECOND.value(),
1229            "Test value should exceed millisecond range"
1230        );
1231    }
1232}
common_sql/convert.rs

common_sql/
convert.rs
