log_query/
log_query.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use chrono::{DateTime, Datelike, Duration, NaiveDate, NaiveTime, TimeZone, Utc};
16use serde::{Deserialize, Serialize};
17use table::table_name::TableName;
18
19use crate::error::{
20    EndBeforeStartSnafu, InvalidDateFormatSnafu, InvalidSpanFormatSnafu, InvalidTimeFilterSnafu,
21    Result,
22};
23
24/// GreptimeDB's log query request.
25#[derive(Debug, Serialize, Deserialize)]
26pub struct LogQuery {
27    // Global query parameters
28    /// A fully qualified table name to query logs from.
29    pub table: TableName,
30    /// Specifies the time range for the log query. See [`TimeFilter`] for more details.
31    pub time_filter: TimeFilter,
32    /// Controls row skipping and fetch on the result set.
33    pub limit: Limit,
34    /// Columns to return in the result set.
35    ///
36    /// The columns can be either from the original log or derived from processing exprs.
37    /// Default (empty) means all columns.
38    ///
39    /// TODO(ruihang): Do we need negative select?
40    pub columns: Vec<String>,
41
42    // Filters
43    /// Conjunction of filters to apply for the raw logs.
44    ///
45    /// Filters here can apply to any LogExpr.
46    pub filters: Filters,
47    /// Adjacent lines to return. Applies to all filters above.
48    ///
49    /// TODO(ruihang): Do we need per-filter context?
50    pub context: Context,
51
52    // Processors
53    /// Expressions to calculate after filter.
54    pub exprs: Vec<LogExpr>,
55}
56
57/// Nested filter structure that supports and/or relationships
58#[derive(Debug, Clone, Serialize, Deserialize)]
59pub enum Filters {
60    /// Single filter condition
61    Single(ColumnFilters),
62    /// Multiple filters with AND relationship
63    And(Vec<Filters>),
64    /// Multiple filters with OR relationship
65    Or(Vec<Filters>),
66    Not(Box<Filters>),
67}
68
69impl Default for Filters {
70    fn default() -> Self {
71        Filters::And(vec![])
72    }
73}
74
75impl From<ColumnFilters> for Filters {
76    fn from(filter: ColumnFilters) -> Self {
77        Filters::Single(filter)
78    }
79}
80
81impl Filters {
82    pub fn and<T: Into<Filters>>(other: Vec<T>) -> Filters {
83        Filters::And(other.into_iter().map(Into::into).collect())
84    }
85
86    pub fn or<T: Into<Filters>>(other: Vec<T>) -> Filters {
87        Filters::Or(other.into_iter().map(Into::into).collect())
88    }
89
90    pub fn single(filter: ColumnFilters) -> Filters {
91        Filters::Single(filter)
92    }
93}
94/// Aggregation function with optional range and alias.
95#[derive(Debug, Clone, Serialize, Deserialize)]
96pub struct AggFunc {
97    /// Function name, e.g., "count", "sum", etc.
98    pub name: String,
99    /// Arguments to the function. e.g., column references or literals. LogExpr::NamedIdent("column1".to_string())
100    pub args: Vec<LogExpr>,
101    pub alias: Option<String>,
102}
103
104impl AggFunc {
105    pub fn new(name: String, args: Vec<LogExpr>, alias: Option<String>) -> Self {
106        Self { name, args, alias }
107    }
108}
109
110/// Expression to calculate on log after filtering.
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub enum LogExpr {
113    NamedIdent(String),
114    PositionalIdent(usize),
115    Literal(String),
116    ScalarFunc {
117        name: String,
118        args: Vec<LogExpr>,
119        alias: Option<String>,
120    },
121    /// Aggregation function with optional grouping.
122    AggrFunc {
123        /// Function name, arguments, and optional alias.
124        expr: Vec<AggFunc>,
125        by: Vec<LogExpr>,
126    },
127    Decompose {
128        expr: Box<LogExpr>,
129        /// JSON, CSV, etc.
130        schema: String,
131        /// Fields with type name to extract from the decomposed value.
132        fields: Vec<(String, String)>,
133    },
134    BinaryOp {
135        left: Box<LogExpr>,
136        op: BinaryOperator,
137        right: Box<LogExpr>,
138    },
139    Alias {
140        expr: Box<LogExpr>,
141        alias: String,
142    },
143    Filter {
144        filter: ColumnFilters,
145    },
146}
147
148impl Default for LogQuery {
149    fn default() -> Self {
150        Self {
151            table: TableName::new("", "", ""),
152            time_filter: Default::default(),
153            filters: Filters::And(vec![]),
154            limit: Limit::default(),
155            context: Default::default(),
156            columns: vec![],
157            exprs: vec![],
158        }
159    }
160}
161
162/// Represents a time range for log query.
163///
164/// This struct allows various formats to express a time range from the user side
165/// for best flexibility:
166/// - Only `start` is provided: the `start` string can be any valid "date" or vaguer
167///     content. For example: "2024-12-01", "2024-12", "2024", etc. It will be treated
168///     as an time range corresponding to the provided date. E.g., "2024-12-01" refers
169///     to the entire 24 hours in that day. In this case, the `start` field cannot be a
170///     timestamp (like "2024-12-01T12:00:00Z").
171/// - Both `start` and `end` are provided: the `start` and `end` strings can be either
172///     a date or a timestamp. The `end` field is exclusive (`[start, end)`). When
173///     `start` is a date it implies the start of the day, and when `end` is a date it
174///     implies the end of the day.
175/// - `span` with `start` OR `end`: the `span` string can be any valid "interval"
176///     For example: "1024s", "1 week", "1 month", etc. The `span` field is applied to
177///     the `start` or `end` field to calculate the other one correspondingly. If `start`
178///     is provided, `end` is calculated as `start + span` and vice versa.
179/// - Only `span` is provided: the `span` string can be any valid "interval" as mentioned
180///     above. In this case, the current time (on the server side) is considered as the `end`.
181/// - All fields are provided: in this case, the `start` and `end` fields are considered
182///     with higher priority, and the `span` field is ignored.
183///
184/// This struct doesn't require a timezone to be presented. When the timezone is not
185/// provided, it will fill the default timezone with the same rules akin to other queries.
186#[derive(Debug, Clone, Default, Serialize, Deserialize)]
187pub struct TimeFilter {
188    pub start: Option<String>,
189    pub end: Option<String>,
190    pub span: Option<String>,
191}
192
193impl TimeFilter {
194    /// Validate and canonicalize the time filter.
195    ///
196    /// This function will try to fill the missing fields and convert all dates to timestamps
197    #[allow(unused_assignments)] // false positive
198    pub fn canonicalize(&mut self) -> Result<()> {
199        let mut start_dt = None;
200        let mut end_dt = None;
201
202        match (&self.start, &self.end, &self.span) {
203            (Some(start), None, None) => {
204                let (start, end_opt) = Self::parse_datetime(start)?;
205                if end_opt.is_none() {
206                    return Err(InvalidTimeFilterSnafu {
207                        filter: self.clone(),
208                    }
209                    .build());
210                }
211                start_dt = Some(start);
212                end_dt = end_opt;
213            }
214            (Some(start), Some(end), _) => {
215                // Both 'start' and 'end' are provided
216                let (start, _) = Self::parse_datetime(start)?;
217                let (end, inferred_end) = Self::parse_datetime(end)?;
218                start_dt = Some(start);
219                end_dt = Some(inferred_end.unwrap_or(end));
220            }
221            (Some(start), None, Some(span)) => {
222                let (start, _) = Self::parse_datetime(start)?;
223                let span = Self::parse_span(span)?;
224                let end = start + span;
225                start_dt = Some(start);
226                end_dt = Some(end);
227            }
228            (None, Some(end), Some(span)) => {
229                let (end, _) = Self::parse_datetime(end)?;
230                let span = Self::parse_span(span)?;
231                let start = end - span;
232                start_dt = Some(start);
233                end_dt = Some(end);
234            }
235            (None, None, Some(span)) => {
236                let span = Self::parse_span(span)?;
237                let end = Utc::now();
238                let start = end - span;
239                start_dt = Some(start);
240                end_dt = Some(end);
241            }
242            _ => {
243                // Exception
244                return Err(InvalidTimeFilterSnafu {
245                    filter: self.clone(),
246                }
247                .build());
248            }
249        }
250
251        // Validate that end is after start
252        if let (Some(start), Some(end)) = (&start_dt, &end_dt)
253            && end <= start
254        {
255            return Err(EndBeforeStartSnafu {
256                start: start.to_rfc3339(),
257                end: end.to_rfc3339(),
258            }
259            .build());
260        }
261
262        // Update the fields with canonicalized timestamps
263        if let Some(start) = start_dt {
264            self.start = Some(start.to_rfc3339());
265        }
266
267        if let Some(end) = end_dt {
268            self.end = Some(end.to_rfc3339());
269        }
270
271        Ok(())
272    }
273
274    /// Util function returns a start and optional end DateTime
275    fn parse_datetime(s: &str) -> Result<(DateTime<Utc>, Option<DateTime<Utc>>)> {
276        if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
277            Ok((dt.with_timezone(&Utc), None))
278        } else {
279            let (naive_date, format) = if let Ok(date) = NaiveDate::parse_from_str(s, "%Y-%m-%d") {
280                (date, "%Y-%m-%d")
281            } else if let Ok(date) = NaiveDate::parse_from_str(&format!("{s}-01"), "%Y-%m-%d") {
282                (date, "%Y-%m")
283            } else if let Ok(date) = NaiveDate::parse_from_str(&format!("{s}-01-01"), "%Y-%m-%d") {
284                (date, "%Y")
285            } else {
286                return Err(InvalidDateFormatSnafu {
287                    input: s.to_string(),
288                }
289                .build());
290            };
291
292            let start = Utc
293                .from_utc_datetime(&naive_date.and_time(NaiveTime::from_hms_opt(0, 0, 0).unwrap()));
294            let end = match format {
295                "%Y-%m-%d" => start + Duration::days(1),
296                "%Y-%m" => {
297                    let next_month = if naive_date.month() == 12 {
298                        NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap()
299                    } else {
300                        NaiveDate::from_ymd_opt(naive_date.year(), naive_date.month() + 1, 1)
301                            .unwrap()
302                    };
303                    Utc.from_utc_datetime(&next_month.and_hms_opt(0, 0, 0).unwrap())
304                }
305                "%Y" => {
306                    let next_year = NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap();
307                    Utc.from_utc_datetime(&next_year.and_hms_opt(0, 0, 0).unwrap())
308                }
309                _ => unreachable!(),
310            };
311            Ok((start, Some(end)))
312        }
313    }
314
315    /// Util function handles durations like "1 week", "1 month", etc (unimplemented).
316    fn parse_span(s: &str) -> Result<Duration> {
317        // Simplified parsing logic
318        if let Ok(seconds) = s.parse::<i64>() {
319            Ok(Duration::seconds(seconds))
320        } else {
321            Err(InvalidSpanFormatSnafu {
322                input: s.to_string(),
323            }
324            .build())
325        }
326    }
327}
328
329/// Represents an expression with filters to query.
330#[derive(Debug, Clone, Serialize, Deserialize)]
331pub struct ColumnFilters {
332    /// Expression to apply filters to. Can be a column reference or any other LogExpr.
333    pub expr: Box<LogExpr>,
334    /// Filters to apply to the expression result. Can be empty.
335    pub filters: Vec<ContentFilter>,
336}
337
338#[derive(Clone, Debug, Serialize, Deserialize)]
339pub enum EqualValue {
340    /// Exact match with a string value.
341    String(String),
342    /// Exact match with a boolean value.
343    Boolean(bool),
344    /// Exact match with a number value.
345    Int(i64),
346    /// Exact match with an unsigned integer value.
347    UInt(u64),
348    /// Exact match with a float value.
349    Float(f64),
350}
351
352impl From<String> for EqualValue {
353    fn from(value: String) -> Self {
354        EqualValue::String(value)
355    }
356}
357
358impl From<bool> for EqualValue {
359    fn from(value: bool) -> Self {
360        EqualValue::Boolean(value)
361    }
362}
363
364impl From<i64> for EqualValue {
365    fn from(value: i64) -> Self {
366        EqualValue::Int(value)
367    }
368}
369
370impl From<f64> for EqualValue {
371    fn from(value: f64) -> Self {
372        EqualValue::Float(value)
373    }
374}
375
376impl From<u64> for EqualValue {
377    fn from(value: u64) -> Self {
378        EqualValue::UInt(value)
379    }
380}
381
382#[derive(Clone, Debug, Serialize, Deserialize)]
383pub enum ContentFilter {
384    // Search-based filters
385    /// Only match the exact content.
386    ///
387    /// For example, if the content is "pale blue dot", the filter "pale" or "pale blue" will match.
388    Exact(String),
389    /// Match the content with a prefix.
390    ///
391    /// For example, if the content is "error message", the filter "err" or "error mess" will match.
392    Prefix(String),
393    /// Match the content with a postfix. Similar to `Prefix`.
394    Postfix(String),
395    /// Match the content with a substring.
396    Contains(String),
397    /// Match the content with a regex pattern. The pattern should be a valid Rust regex.
398    Regex(String),
399
400    // Value-based filters
401    /// Content exists, a.k.a. not null.
402    Exist,
403    Between {
404        start: String,
405        end: String,
406        start_inclusive: bool,
407        end_inclusive: bool,
408    },
409    GreatThan {
410        value: String,
411        inclusive: bool,
412    },
413    LessThan {
414        value: String,
415        inclusive: bool,
416    },
417    In(Vec<String>),
418    IsTrue,
419    IsFalse,
420    Equal(EqualValue),
421
422    // Compound filters
423    Compound(Vec<ContentFilter>, ConjunctionOperator),
424}
425
426#[derive(Clone, Debug, Serialize, Deserialize)]
427pub enum ConjunctionOperator {
428    And,
429    Or,
430}
431
432/// Binary operators for LogExpr::BinaryOp.
433#[derive(Clone, Debug, Serialize, Deserialize)]
434pub enum BinaryOperator {
435    // Comparison operators
436    Eq,
437    Ne,
438    Lt,
439    Le,
440    Gt,
441    Ge,
442
443    // Arithmetic operators
444    Plus,
445    Minus,
446    Multiply,
447    Divide,
448    Modulo,
449
450    // Logical operators
451    And,
452    Or,
453}
454
455/// Controls how many adjacent lines to return.
456#[derive(Debug, Default, Serialize, Deserialize)]
457pub enum Context {
458    #[default]
459    None,
460    /// Specify the number of lines before and after the matched line separately.
461    Lines(usize, usize),
462    /// Specify the number of seconds before and after the matched line occurred.
463    Seconds(usize, usize),
464}
465
466/// Represents limit and offset parameters for query pagination.
467#[derive(Debug, Default, Serialize, Deserialize)]
468pub struct Limit {
469    /// Optional number of items to skip before starting to return results
470    pub skip: Option<usize>,
471    /// Optional number of items to return after skipping
472    pub fetch: Option<usize>,
473}
474
475#[cfg(test)]
476mod tests {
477    use super::*;
478    use crate::error::Error;
479
480    #[test]
481    fn test_canonicalize() {
482        // with 'start' only
483        let mut tf = TimeFilter {
484            start: Some("2023-10-01".to_string()),
485            end: None,
486            span: None,
487        };
488        tf.canonicalize().unwrap();
489        assert!(tf.end.is_some());
490
491        // with 'start' and 'span'
492        let mut tf = TimeFilter {
493            start: Some("2023-10-01T00:00:00Z".to_string()),
494            end: None,
495            span: Some("86400".to_string()), // 1 day in seconds
496        };
497        tf.canonicalize().unwrap();
498        assert_eq!(tf.end.as_ref().unwrap(), "2023-10-02T00:00:00+00:00");
499
500        // with 'end' and 'span'
501        let mut tf = TimeFilter {
502            start: None,
503            end: Some("2023-10-02T00:00:00Z".to_string()),
504            span: Some("86400".to_string()), // 1 day in seconds
505        };
506        tf.canonicalize().unwrap();
507        assert_eq!(tf.start.as_ref().unwrap(), "2023-10-01T00:00:00+00:00");
508
509        // with both 'start' and 'end'
510        let mut tf = TimeFilter {
511            start: Some("2023-10-01T00:00:00Z".to_string()),
512            end: Some("2023-10-02T00:00:00Z".to_string()),
513            span: None,
514        };
515        tf.canonicalize().unwrap();
516        assert_eq!(tf.start.as_ref().unwrap(), "2023-10-01T00:00:00+00:00");
517        assert_eq!(tf.end.as_ref().unwrap(), "2023-10-02T00:00:00+00:00");
518
519        // with invalid date format
520        let mut tf = TimeFilter {
521            start: Some("invalid-date".to_string()),
522            end: None,
523            span: None,
524        };
525        let result = tf.canonicalize();
526        assert!(matches!(result, Err(Error::InvalidDateFormat { .. })));
527
528        // with missing 'start' and 'end'
529        let mut tf = TimeFilter {
530            start: None,
531            end: None,
532            span: None,
533        };
534        let result = tf.canonicalize();
535        assert!(matches!(result, Err(Error::InvalidTimeFilter { .. })));
536
537        // 'end' is before 'start'
538        let mut tf = TimeFilter {
539            start: Some("2023-10-02T00:00:00Z".to_string()),
540            end: Some("2023-10-01T00:00:00Z".to_string()),
541            span: None,
542        };
543        let result = tf.canonicalize();
544        assert!(matches!(result, Err(Error::EndBeforeStart { .. })));
545    }
546
547    #[test]
548    fn test_canonicalize_date_end() {
549        let mut date_end = TimeFilter {
550            start: Some("2023-10-01".to_string()),
551            end: Some("2023-10-02".to_string()),
552            span: None,
553        };
554        date_end.canonicalize().unwrap();
555        assert_eq!(date_end.end.as_deref(), Some("2023-10-03T00:00:00+00:00"));
556
557        let mut timestamp_end = TimeFilter {
558            start: Some("2023-10-01".to_string()),
559            end: Some("2023-10-02T12:34:56Z".to_string()),
560            span: None,
561        };
562        timestamp_end.canonicalize().unwrap();
563        assert_eq!(
564            timestamp_end.end.as_deref(),
565            Some("2023-10-02T12:34:56+00:00")
566        );
567
568        let mut month = TimeFilter {
569            start: Some("2023-10".to_string()),
570            end: Some("2023-10".to_string()),
571            span: None,
572        };
573        month.canonicalize().unwrap();
574        assert_eq!(month.start.as_deref(), Some("2023-10-01T00:00:00+00:00"));
575        assert_eq!(month.end.as_deref(), Some("2023-11-01T00:00:00+00:00"));
576
577        let mut year = TimeFilter {
578            start: Some("2023".to_string()),
579            end: Some("2023".to_string()),
580            span: None,
581        };
582        year.canonicalize().unwrap();
583        assert_eq!(year.start.as_deref(), Some("2023-01-01T00:00:00+00:00"));
584        assert_eq!(year.end.as_deref(), Some("2024-01-01T00:00:00+00:00"));
585    }
586}
log_query/log_query.rs

log_query/
log_query.rs
