1use std::collections::HashMap;
16
17use chrono::{DateTime, Utc};
18use common_base::memory_limit::MemoryLimit;
19use serde::{Deserialize, Serialize};
20use session::context::QueryContextRef;
21use store_api::storage::RegionId;
22use table::metadata::TableId;
23
24use crate::error::{Error, InvalidQueryContextExtensionSnafu, Result};
25
26pub const FLOW_INCREMENTAL_AFTER_SEQS: &str = "flow.incremental_after_seqs";
27pub const FLOW_INCREMENTAL_MODE: &str = "flow.incremental_mode";
28pub const FLOW_RETURN_REGION_SEQ: &str = "flow.return_region_seq";
29pub const FLOW_SINK_TABLE_ID: &str = "flow.sink_table_id";
30pub const FLOW_SCHEDULED_TIME_MILLIS: &str = "flow.scheduled_time_millis";
34pub const QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN: &str =
36 "query.enable_remote_dynamic_filter_pushdown";
37
38pub const FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY: &str = "memtable_only";
39
40#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
42#[serde(default)]
43pub struct QueryOptions {
44 pub parallelism: usize,
46 pub allow_query_fallback: bool,
48 pub memory_pool_size: MemoryLimit,
52 #[serde(skip)]
54 pub enable_per_region_metrics: bool,
55}
56
57#[allow(clippy::derivable_impls)]
58impl Default for QueryOptions {
59 fn default() -> Self {
60 Self {
61 parallelism: 0,
62 allow_query_fallback: false,
63 memory_pool_size: MemoryLimit::default(),
64 enable_per_region_metrics: false,
65 }
66 }
67}
68
69#[derive(Debug, Clone, Copy, PartialEq, Eq)]
70pub enum FlowIncrementalMode {
71 MemtableOnly,
72}
73
74#[derive(Debug, Clone, PartialEq, Eq, Default)]
75pub struct FlowQueryExtensions {
76 pub incremental_after_seqs: Option<HashMap<u64, u64>>,
78 pub incremental_mode: Option<FlowIncrementalMode>,
80 pub return_region_seq: bool,
82 pub sink_table_id: Option<TableId>,
84}
85
86impl FlowQueryExtensions {
87 pub fn parse_flow_extensions(extensions: &HashMap<String, String>) -> Result<Option<Self>> {
93 let has_flow_context = extensions.contains_key(FLOW_INCREMENTAL_AFTER_SEQS)
94 || extensions.contains_key(FLOW_INCREMENTAL_MODE)
95 || extensions.contains_key(FLOW_RETURN_REGION_SEQ)
96 || extensions.contains_key(FLOW_SINK_TABLE_ID);
97
98 if !has_flow_context {
99 return Ok(None);
100 }
101
102 let incremental_mode = extensions
103 .get(FLOW_INCREMENTAL_MODE)
104 .map(|value| match value.as_str() {
105 v if v.eq_ignore_ascii_case(FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY) => {
106 Ok(FlowIncrementalMode::MemtableOnly)
107 }
108 _ => Err(invalid_query_context_extension(format!(
109 "Invalid value for {}: {}",
110 FLOW_INCREMENTAL_MODE, value
111 ))),
112 })
113 .transpose()?;
114
115 let incremental_after_seqs = extensions
116 .get(FLOW_INCREMENTAL_AFTER_SEQS)
117 .map(|value| parse_incremental_after_seqs(value.as_str()))
118 .transpose()?;
119
120 let return_region_seq = extensions
121 .get(FLOW_RETURN_REGION_SEQ)
122 .map(|value| parse_bool(FLOW_RETURN_REGION_SEQ, value.as_str()))
123 .transpose()?
124 .unwrap_or(false);
125
126 let sink_table_id = extensions
127 .get(FLOW_SINK_TABLE_ID)
128 .map(|value| {
129 value.parse::<TableId>().map_err(|_| {
130 invalid_query_context_extension(format!(
131 "Invalid value for {}: {}",
132 FLOW_SINK_TABLE_ID, value
133 ))
134 })
135 })
136 .transpose()?;
137
138 if matches!(incremental_mode, Some(FlowIncrementalMode::MemtableOnly)) {
139 let after_seqs = incremental_after_seqs.as_ref().ok_or_else(|| {
140 invalid_query_context_extension(format!(
141 "{} is required when {}={}.",
142 FLOW_INCREMENTAL_AFTER_SEQS,
143 FLOW_INCREMENTAL_MODE,
144 FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY
145 ))
146 })?;
147 if after_seqs.is_empty() {
148 return Err(invalid_query_context_extension(format!(
149 "{} must not be empty when {}={}.",
150 FLOW_INCREMENTAL_AFTER_SEQS,
151 FLOW_INCREMENTAL_MODE,
152 FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY
153 )));
154 }
155 }
156
157 Ok(Some(Self {
158 incremental_after_seqs,
159 incremental_mode,
160 return_region_seq,
161 sink_table_id,
162 }))
163 }
164
165 pub fn validate_for_scan(&self, source_region_id: RegionId) -> Result<bool> {
166 if self.sink_table_id.is_some() && self.sink_table_id == Some(source_region_id.table_id()) {
167 return Ok(false);
168 }
169
170 if matches!(
171 self.incremental_mode,
172 Some(FlowIncrementalMode::MemtableOnly)
173 ) {
174 let after_seqs = self.incremental_after_seqs.as_ref().ok_or_else(|| {
175 invalid_query_context_extension(format!(
176 "{} is required when {}=memtable_only.",
177 FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE
178 ))
179 })?;
180
181 if !after_seqs.contains_key(&source_region_id.as_u64()) {
182 return Err(invalid_query_context_extension(format!(
183 "Missing region {} in {} when {}=memtable_only.",
184 source_region_id, FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE
185 )));
186 }
187 }
188
189 Ok(self.incremental_after_seqs.is_some())
190 }
191
192 pub fn should_collect_region_watermark(&self) -> bool {
193 should_collect_region_watermark(
194 self.return_region_seq,
195 self.incremental_after_seqs.is_some(),
196 )
197 }
198}
199
200pub fn remote_dyn_filter_pushdown_enabled_from_extensions(
207 extensions: &HashMap<String, String>,
208) -> Result<bool> {
209 extensions
210 .get(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN)
211 .map(|value| parse_bool(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN, value.as_str()))
212 .transpose()
213 .map(|value| value.unwrap_or(true))
214}
215
216pub fn should_collect_region_watermark_from_extensions(
221 extensions: &HashMap<String, String>,
222) -> bool {
223 let return_region_seq = extensions
224 .get(FLOW_RETURN_REGION_SEQ)
225 .is_some_and(|value| value.eq_ignore_ascii_case("true"));
226 let has_incremental_after_seqs = extensions.contains_key(FLOW_INCREMENTAL_AFTER_SEQS);
227
228 should_collect_region_watermark(return_region_seq, has_incremental_after_seqs)
229}
230
231fn should_collect_region_watermark(
232 return_region_seq: bool,
233 has_incremental_after_seqs: bool,
234) -> bool {
235 return_region_seq || has_incremental_after_seqs
236}
237
238fn parse_incremental_after_seqs(value: &str) -> Result<HashMap<u64, u64>> {
239 let raw = serde_json::from_str::<HashMap<String, serde_json::Value>>(value).map_err(|e| {
240 invalid_query_context_extension(format!(
241 "Invalid JSON for {}: {} ({})",
242 FLOW_INCREMENTAL_AFTER_SEQS, value, e
243 ))
244 })?;
245
246 raw.into_iter()
247 .map(|(region_id, raw_seq)| {
248 let region_id = region_id.parse::<u64>().map_err(|_| {
249 invalid_query_context_extension(format!(
250 "Invalid region id in {}: {}",
251 FLOW_INCREMENTAL_AFTER_SEQS, region_id
252 ))
253 })?;
254
255 let seq = match raw_seq {
256 serde_json::Value::Number(num) => num.as_u64().ok_or_else(|| {
257 invalid_query_context_extension(format!(
258 "Invalid sequence value in {} for region {}: {}",
259 FLOW_INCREMENTAL_AFTER_SEQS, region_id, num
260 ))
261 })?,
262 serde_json::Value::String(s) => s.parse::<u64>().map_err(|_| {
263 invalid_query_context_extension(format!(
264 "Invalid sequence string in {} for region {}: {}",
265 FLOW_INCREMENTAL_AFTER_SEQS, region_id, s
266 ))
267 })?,
268 _ => {
269 return Err(invalid_query_context_extension(format!(
270 "Invalid sequence value type in {} for region {}",
271 FLOW_INCREMENTAL_AFTER_SEQS, region_id
272 )));
273 }
274 };
275
276 Ok((region_id, seq))
277 })
278 .collect()
279}
280
281fn parse_bool(option_name: &str, value: &str) -> Result<bool> {
282 match value {
283 v if v.eq_ignore_ascii_case("true") => Ok(true),
284 v if v.eq_ignore_ascii_case("false") => Ok(false),
285 _ => Err(invalid_query_context_extension(format!(
286 "Invalid value for {}: {}",
287 option_name, value
288 ))),
289 }
290}
291
292pub fn parse_scheduled_time_millis(extensions: &HashMap<String, String>) -> Result<Option<i64>> {
297 match extensions.get(FLOW_SCHEDULED_TIME_MILLIS) {
298 Some(val) => val.parse::<i64>().map(Some).map_err(|_| {
299 invalid_query_context_extension(format!(
300 "Invalid value for {}: {}",
301 FLOW_SCHEDULED_TIME_MILLIS, val
302 ))
303 }),
304 None => Ok(None),
305 }
306}
307
308pub fn parse_scheduled_time_datetime(
314 extensions: &HashMap<String, String>,
315) -> Result<Option<DateTime<Utc>>> {
316 match parse_scheduled_time_millis(extensions)? {
317 Some(millis) => DateTime::from_timestamp_millis(millis)
318 .map(Some)
319 .ok_or_else(|| {
320 invalid_query_context_extension(format!(
321 "Out-of-range timestamp for {}: {} ms",
322 FLOW_SCHEDULED_TIME_MILLIS, millis
323 ))
324 }),
325 None => Ok(None),
326 }
327}
328
329pub fn scheduled_time_from_ctx(query_ctx: &QueryContextRef) -> Option<DateTime<Utc>> {
334 let extensions = query_ctx.extensions();
335 parse_scheduled_time_datetime(&extensions).ok().flatten()
336}
337
338fn invalid_query_context_extension(reason: String) -> Error {
339 InvalidQueryContextExtensionSnafu { reason }.build()
340}
341
342#[cfg(test)]
343mod flow_extension_tests {
344 use super::*;
345
346 #[test]
347 fn test_parse_flow_extensions_returns_none_for_non_flow_query() {
348 let exts = HashMap::new();
349 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap();
350
351 assert_eq!(parsed, None);
352 }
353
354 #[test]
355 fn test_remote_dyn_filter_pushdown_enabled_from_extensions_defaults_true() {
356 assert!(remote_dyn_filter_pushdown_enabled_from_extensions(&HashMap::new()).unwrap());
357 }
358
359 #[test]
360 fn test_remote_dyn_filter_pushdown_enabled_from_extensions_parses_bool() {
361 let exts = HashMap::from([(
362 QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN.to_string(),
363 "false".to_string(),
364 )]);
365 assert!(!remote_dyn_filter_pushdown_enabled_from_extensions(&exts).unwrap());
366
367 let exts = HashMap::from([(
368 QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN.to_string(),
369 "true".to_string(),
370 )]);
371 assert!(remote_dyn_filter_pushdown_enabled_from_extensions(&exts).unwrap());
372 }
373
374 #[test]
375 fn test_remote_dyn_filter_pushdown_enabled_from_extensions_rejects_invalid_bool() {
376 let exts = HashMap::from([(
377 QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN.to_string(),
378 "invalid".to_string(),
379 )]);
380
381 let err = remote_dyn_filter_pushdown_enabled_from_extensions(&exts).unwrap_err();
382 assert!(format!("{err}").contains(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN));
383 }
384
385 #[test]
386 fn test_parse_flow_extensions_memtable_only_success() {
387 let exts = HashMap::from([
388 (
389 FLOW_INCREMENTAL_MODE.to_string(),
390 FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY.to_string(),
391 ),
392 (
393 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
394 r#"{"1":10,"2":20}"#.to_string(),
395 ),
396 (FLOW_RETURN_REGION_SEQ.to_string(), "true".to_string()),
397 (FLOW_SINK_TABLE_ID.to_string(), "1024".to_string()),
398 ]);
399
400 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts)
401 .unwrap()
402 .unwrap();
403 assert_eq!(
404 parsed.incremental_mode,
405 Some(FlowIncrementalMode::MemtableOnly)
406 );
407 assert_eq!(
408 parsed.incremental_after_seqs.unwrap(),
409 HashMap::from([(1, 10), (2, 20)])
410 );
411 assert!(parsed.return_region_seq);
412 assert_eq!(parsed.sink_table_id, Some(1024));
413 }
414
415 #[test]
416 fn test_parse_flow_extensions_mode_requires_after_seqs() {
417 let exts = HashMap::from([(
418 FLOW_INCREMENTAL_MODE.to_string(),
419 FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY.to_string(),
420 )]);
421
422 let err = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap_err();
423 assert!(format!("{err}").contains(FLOW_INCREMENTAL_AFTER_SEQS));
424 }
425
426 #[test]
427 fn test_parse_flow_extensions_invalid_mode() {
428 let exts = HashMap::from([(FLOW_INCREMENTAL_MODE.to_string(), "foo".to_string())]);
429
430 let err = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap_err();
431 assert!(format!("{err}").contains(FLOW_INCREMENTAL_MODE));
432 }
433
434 #[test]
435 fn test_parse_flow_extensions_invalid_after_seqs_json() {
436 let exts = HashMap::from([
437 (
438 FLOW_INCREMENTAL_MODE.to_string(),
439 FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY.to_string(),
440 ),
441 (
442 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
443 "not-json".to_string(),
444 ),
445 ]);
446
447 let err = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap_err();
448 assert!(format!("{err}").contains(FLOW_INCREMENTAL_AFTER_SEQS));
449 }
450
451 #[test]
452 fn test_parse_flow_extensions_after_seqs_string_values() {
453 let exts = HashMap::from([(
454 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
455 r#"{"1":"10","2":"20"}"#.to_string(),
456 )]);
457
458 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts)
459 .unwrap()
460 .unwrap();
461 assert_eq!(
462 parsed.incremental_after_seqs.unwrap(),
463 HashMap::from([(1, 10), (2, 20)])
464 );
465 }
466
467 #[test]
468 fn test_parse_flow_extensions_after_seqs_invalid_value_type() {
469 let exts = HashMap::from([(
470 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
471 r#"{"1":true}"#.to_string(),
472 )]);
473
474 let err = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap_err();
475 assert!(format!("{err}").contains(FLOW_INCREMENTAL_AFTER_SEQS));
476 }
477
478 #[test]
479 fn test_parse_flow_extensions_invalid_sink_table_id() {
480 let exts = HashMap::from([(FLOW_SINK_TABLE_ID.to_string(), "x".to_string())]);
481
482 let err = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap_err();
483 assert!(format!("{err}").contains(FLOW_SINK_TABLE_ID));
484 }
485
486 #[test]
487 fn test_validate_for_scan_missing_source_region() {
488 let source_region_id = RegionId::new(100, 2);
489 let existing_region_id = RegionId::new(100, 1);
490 let exts = HashMap::from([
491 (
492 FLOW_INCREMENTAL_MODE.to_string(),
493 FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY.to_string(),
494 ),
495 (
496 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
497 format!(r#"{{"{}":10}}"#, existing_region_id.as_u64()),
498 ),
499 ]);
500
501 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts)
502 .unwrap()
503 .unwrap();
504 let err = parsed.validate_for_scan(source_region_id).unwrap_err();
505 assert!(format!("{err}").contains("Missing region"));
506 }
507
508 #[test]
509 fn test_validate_for_scan_sink_table_excluded() {
510 let source_region_id = RegionId::new(1024, 1);
511 let exts = HashMap::from([
512 (
513 FLOW_INCREMENTAL_MODE.to_string(),
514 FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY.to_string(),
515 ),
516 (
517 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
518 format!(r#"{{"{}":10}}"#, source_region_id.as_u64()),
519 ),
520 (FLOW_SINK_TABLE_ID.to_string(), "1024".to_string()),
521 ]);
522
523 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts)
524 .unwrap()
525 .unwrap();
526 let apply_incremental = parsed.validate_for_scan(source_region_id).unwrap();
527 assert!(!apply_incremental);
528 }
529
530 #[test]
531 fn test_should_collect_region_watermark_defaults_false() {
532 let parsed = FlowQueryExtensions::default();
533 assert!(!parsed.should_collect_region_watermark());
534 }
535
536 #[test]
537 fn test_should_collect_region_watermark_true_for_return_region_seq() {
538 let parsed = FlowQueryExtensions {
539 return_region_seq: true,
540 ..Default::default()
541 };
542 assert!(parsed.should_collect_region_watermark());
543 }
544
545 #[test]
546 fn test_should_collect_region_watermark_true_for_incremental_query() {
547 let parsed = FlowQueryExtensions {
548 incremental_after_seqs: Some(HashMap::from([(1, 10)])),
549 ..Default::default()
550 };
551 assert!(parsed.should_collect_region_watermark());
552 }
553
554 #[test]
555 fn test_should_collect_region_watermark_from_extensions() {
556 let exts = HashMap::from([(FLOW_RETURN_REGION_SEQ.to_string(), "true".to_string())]);
557 assert!(should_collect_region_watermark_from_extensions(&exts));
558
559 let exts = HashMap::from([(
560 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
561 r#"{"1":10}"#.to_string(),
562 )]);
563 assert!(should_collect_region_watermark_from_extensions(&exts));
564
565 let exts = HashMap::from([(FLOW_RETURN_REGION_SEQ.to_string(), "false".to_string())]);
566 assert!(!should_collect_region_watermark_from_extensions(&exts));
567 assert!(!should_collect_region_watermark_from_extensions(
568 &HashMap::new()
569 ));
570 }
571
572 #[test]
573 fn test_parse_flow_extensions_return_region_seq_only_returns_some() {
574 let exts = HashMap::from([(FLOW_RETURN_REGION_SEQ.to_string(), "true".to_string())]);
575
576 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts)
577 .unwrap()
578 .unwrap();
579
580 assert!(parsed.return_region_seq);
581 }
582
583 #[test]
584 fn test_parse_flow_extensions_sink_table_only_returns_some() {
585 let exts = HashMap::from([(FLOW_SINK_TABLE_ID.to_string(), "1024".to_string())]);
586
587 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts)
588 .unwrap()
589 .unwrap();
590
591 assert_eq!(parsed.sink_table_id, Some(1024));
592 }
593
594 #[test]
595 fn test_parse_flow_extensions_incremental_after_seqs_only_returns_some() {
596 let exts = HashMap::from([(
597 FLOW_INCREMENTAL_AFTER_SEQS.to_string(),
598 r#"{"1":10}"#.to_string(),
599 )]);
600
601 let parsed = FlowQueryExtensions::parse_flow_extensions(&exts)
602 .unwrap()
603 .unwrap();
604
605 assert_eq!(
606 parsed.incremental_after_seqs,
607 Some(HashMap::from([(1, 10)]))
608 );
609 }
610
611 #[test]
614 fn test_parse_scheduled_time_millis_absent() {
615 let exts = HashMap::new();
616 assert_eq!(parse_scheduled_time_millis(&exts).unwrap(), None);
617 }
618
619 #[test]
620 fn test_parse_scheduled_time_millis_valid() {
621 let exts = HashMap::from([(
622 FLOW_SCHEDULED_TIME_MILLIS.to_string(),
623 "1700000000000".to_string(),
624 )]);
625 assert_eq!(
626 parse_scheduled_time_millis(&exts).unwrap(),
627 Some(1700000000000)
628 );
629 }
630
631 #[test]
632 fn test_parse_scheduled_time_millis_malformed() {
633 let exts = HashMap::from([(
634 FLOW_SCHEDULED_TIME_MILLIS.to_string(),
635 "not-a-number".to_string(),
636 )]);
637 let err = parse_scheduled_time_millis(&exts).unwrap_err();
638 assert!(format!("{err}").contains(FLOW_SCHEDULED_TIME_MILLIS));
639 }
640
641 #[test]
642 fn test_parse_scheduled_time_datetime_valid() {
643 let exts = HashMap::from([(
644 FLOW_SCHEDULED_TIME_MILLIS.to_string(),
645 "1700000000000".to_string(),
646 )]);
647 let dt = parse_scheduled_time_datetime(&exts).unwrap().unwrap();
648 assert_eq!(dt.timestamp_millis(), 1700000000000);
649 }
650
651 #[test]
652 fn test_parse_scheduled_time_datetime_negative_millis() {
653 let exts = HashMap::from([(FLOW_SCHEDULED_TIME_MILLIS.to_string(), "-1".to_string())]);
654 let dt = parse_scheduled_time_datetime(&exts).unwrap().unwrap();
655 assert_eq!(dt.timestamp_millis(), -1);
656 }
657
658 #[test]
659 fn test_parse_scheduled_time_datetime_out_of_range() {
660 let exts = HashMap::from([(FLOW_SCHEDULED_TIME_MILLIS.to_string(), i64::MAX.to_string())]);
662 let err = parse_scheduled_time_datetime(&exts).unwrap_err();
663 assert!(format!("{err}").contains("Out-of-range"));
664 }
665}